Apache DataFusion SQL Query Engine
feat: add an async API for spill file writing (#24923)
Phoenix6f4cac32937e
diff --git a/datafusion-examples/examples/data_io/object_store_spill.rs b/datafusion-examples/examples/data_io/object_store_spill.rsindex d7d5392f66..c714c45570 100644--- a/datafusion-examples/examples/data_io/object_store_spill.rs+++ b/datafusion-examples/examples/data_io/object_store_spill.rs@@ -24,26 +24,31 @@//! See [`datafusion::execution::memory_pool`] for more information on how//! DataFusion decides when operators should spill, and [`SpillFile`] for the//! spill file abstraction this example implements.-use std::future::Future;-use std::io::Write;+//!+//! This example exercises the asynchronous external-sort spill path. Execution+//! paths that require a partially written local file to be readable still use+//! the synchronous spill API.use std::path::Path as StdPath;use std::pin::Pin;use std::sync::Arc;use std::sync::atomic::{AtomicU64, Ordering};+use async_trait::async_trait;use bytes::Bytes;-use datafusion::common::Result;+use datafusion::common::{Result, not_impl_err};use datafusion::execution::disk_manager::DiskManagerBuilder;use datafusion::execution::runtime_env::RuntimeEnvBuilder;-use datafusion::execution::{SpillFile, SpillWriter, TempFileFactory};+use datafusion::execution::{AsyncSpillWriter, SpillFile, SpillWriter, TempFileFactory};use datafusion::prelude::{SessionConfig, SessionContext};-use datafusion_common::exec_err;use futures::{Stream, StreamExt, TryStreamExt, stream};use object_store::local::LocalFileSystem;use object_store::path::Path;-use object_store::{ObjectStore, ObjectStoreExt, PutPayload};+use object_store::{MultipartUpload, ObjectStore, ObjectStoreExt, PutPayloadMut};use tempfile::tempdir;+/// Most remote object stores require non-final multipart parts to be at least 5 MiB.+const OBJECT_STORE_PART_SIZE: usize = 5 * 1024 * 1024;+/// Demonstrates configuring DataFusion with spill files backed by an ObjectStore.pub async fn object_store_spill() -> Result<()> {// A real system would use S3, GCS, Azure, or some other ObjectStore for@@ -54,7 +59,7 @@ pub async fn object_store_spill() -> Result<()> {Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?);// Create the custom TempFileFactory that creates spill files in the ObjectStore.- let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(store));+ let temp_file_factory = Arc::new(ObjectStoreTempFileFactory::new(Arc::clone(&store)));let disk_manager_builder =DiskManagerBuilder::default().with_temp_file_factory(temp_file_factory.clone());let runtime = RuntimeEnvBuilder::new()@@ -94,6 +99,19 @@ pub async fn object_store_spill() -> Result<()> {temp_file_factory.created_files() > 0,"expected the custom TempFileFactory to be used for spilling");+ // Ensure the workload crosses a multipart boundary so the example uploads+ // at least one complete part before the final `finish` call.+ let spill_prefix = Path::from("spill");+ let spill_objects = store+ .list(Some(&spill_prefix))+ .try_collect::<Vec<_>>()+ .await?;+ assert!(+ spill_objects+ .iter()+ .any(|object| object.size >= OBJECT_STORE_PART_SIZE as u64),+ "expected at least one spill object to exceed the multipart part size"+ );Ok(())}@@ -154,8 +172,8 @@ impl TempFileFactory for ObjectStoreTempFileFactory {/// Logical spill file stored at an ObjectStore path.///-/// DataFusion writes spill data by calling [`SpillFile::open_writer`] and reads-/// it back by calling [`SpillFile::read_stream`].+/// DataFusion writes spill data by calling [`SpillFile::open_async_writer`] and+/// reads it back by calling [`SpillFile::read_stream`].struct ObjectStoreSpillFile {/// ObjectStore containing the spill object.store: Arc<dyn ObjectStore>,@@ -165,6 +183,7 @@ struct ObjectStoreSpillFile {size: Arc<AtomicU64>,}+#[async_trait]impl SpillFile for ObjectStoreSpillFile {/// Return no local filesystem path because the spill file is accessed through ObjectStore.fn path(&self) -> Option<&StdPath> {@@ -193,81 +212,180 @@ impl SpillFile for ObjectStoreSpillFile {Ok(Box::pin(stream))}- /// Open a synchronous writer for this spill file.+ /// This example backend supports the asynchronous external-sort spill path only.fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {- // Create a writer that buffers bytes and uploads them on finish.+ not_impl_err!("Synchronous spill writing is not supported by this backend")+ }++ /// Open an asynchronous, multipart-capable writer for this spill file.+ async fn open_async_writer(&self) -> Result<Box<dyn AsyncSpillWriter>> {+ let upload = self.store.put_multipart(&self.location).await?;Ok(Box::new(ObjectStoreSpillWriter {- store: Arc::clone(&self.store),- location: self.location.clone(),+ upload,+ buffer: PutPayloadMut::new(),size: Arc::clone(&self.size),- buffer: Vec::new(),+ bytes_written: 0,}))}}-/// Adapts DataFusion's [`SpillWriter`] API to ObjectStore.-///-/// This simple example buffers bytes in memory and uploads them in-/// [`SpillWriter::finish`]. A production remote implementation should consider-/// multipart or streaming uploads.+/// Adapts DataFusion's [`AsyncSpillWriter`] API to ObjectStore.struct ObjectStoreSpillWriter {- /// ObjectStore to read/write bytes to.- store: Arc<dyn ObjectStore>,- /// ObjectStore path to upload to.- location: Path,+ /// Multipart upload used to stream completed parts to the store.+ upload: Box<dyn MultipartUpload>,+ /// Buffers at most one part while preserving owned `Bytes` chunks.+ buffer: PutPayloadMut,/// Shared size field on the corresponding [`ObjectStoreSpillFile`].size: Arc<AtomicU64>,- /// Buffered spill bytes waiting to be uploaded.- ///- /// This simple example buffers the spill and uploads it on finish.- /// Production remote stores should consider multipart or streaming uploads.- buffer: Vec<u8>,+ /// Number of bytes passed to the writer.+ bytes_written: u64,+}++impl ObjectStoreSpillWriter {+ async fn flush_part(&mut self) -> object_store::Result<()> {+ if self.buffer.is_empty() {+ return Ok(());+ }++ let part = std::mem::take(&mut self.buffer).freeze();+ self.upload.put_part(part).await+ }}-impl Write for ObjectStoreSpillWriter {- /// Append bytes to the in-memory buffer.- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {- // Buffer bytes written through the synchronous Write API.- self.buffer.extend_from_slice(buf);- Ok(buf.len())+#[async_trait]+impl AsyncSpillWriter for ObjectStoreSpillWriter {+ async fn write_all(&mut self, mut data: Bytes) -> Result<()> {+ let len = data.len() as u64;+ while !data.is_empty() {+ let remaining = OBJECT_STORE_PART_SIZE - self.buffer.content_length();+ if data.len() < remaining {+ // A Bytes slice can pin an entire Arrow allocation. Copy only+ // the tail retained after this call so it remains accurately+ // bounded after DataFusion releases the batch reservation.+ self.buffer.push(Bytes::copy_from_slice(&data));+ break;+ }++ self.buffer.push(data.split_to(remaining));+ self.flush_part().await?;+ }+ self.bytes_written += len;+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ self.flush_part().await?;+ self.upload.complete().await?;+ self.size.store(self.bytes_written, Ordering::Relaxed);+ Ok(())}- /// No-op because data is committed in [`SpillWriter::finish`].- fn flush(&mut self) -> std::io::Result<()> {+ async fn abort(&mut self) -> Result<()> {+ // Release locally buffered spill data before waiting on remote cleanup.+ self.buffer = PutPayloadMut::new();+ self.upload.abort().await?;Ok(())}}-impl SpillWriter for ObjectStoreSpillWriter {- /// Upload buffered bytes to ObjectStore and mark the spill file complete.- fn finish(&mut self) -> Result<()> {- // Move the buffered bytes into the upload future.- let store = Arc::clone(&self.store);- let location = self.location.clone();- let data = std::mem::take(&mut self.buffer);- let size = data.len() as u64;-- // This simple example buffers the spill and uploads it on finish.- // Production remote stores should consider multipart or streaming uploads.- block_on_object_store(async move {- store- .put(&location, PutPayload::from_bytes(data.into()))- .await?;+#[cfg(test)]+mod tests {+ use super::*;+ use object_store::{PutPayload, PutResult, UploadPart};+ use std::sync::Mutex;+ use std::sync::atomic::AtomicBool;++ #[derive(Debug)]+ struct RecordingMultipartUpload {+ part_sizes: Arc<Mutex<Vec<usize>>>,+ }++ #[async_trait]+ impl MultipartUpload for RecordingMultipartUpload {+ fn put_part(&mut self, data: PutPayload) -> UploadPart {+ self.part_sizes.lock().unwrap().push(data.content_length());+ Box::pin(async { Ok(()) })+ }++ async fn complete(&mut self) -> object_store::Result<PutResult> {+ Ok(PutResult {+ e_tag: None,+ version: None,+ })+ }++ async fn abort(&mut self) -> object_store::Result<()> {Ok(())- })?;+ }+ }++ struct DropTrackingOwner {+ data: Vec<u8>,+ dropped: Arc<AtomicBool>,+ }++ impl AsRef<[u8]> for DropTrackingOwner {+ fn as_ref(&self) -> &[u8] {+ &self.data+ }+ }++ impl Drop for DropTrackingOwner {+ fn drop(&mut self) {+ self.dropped.store(true, Ordering::Relaxed);+ }+ }++ #[tokio::test]+ async fn uploads_full_part_before_finish() -> Result<()> {+ let part_sizes = Arc::new(Mutex::new(Vec::new()));+ let upload = RecordingMultipartUpload {+ part_sizes: Arc::clone(&part_sizes),+ };+ let mut writer = ObjectStoreSpillWriter {+ upload: Box::new(upload),+ buffer: PutPayloadMut::new(),+ size: Arc::new(AtomicU64::new(0)),+ bytes_written: 0,+ };- self.size.store(size, Ordering::Relaxed);+ writer+ .write_all(Bytes::from(vec![42; OBJECT_STORE_PART_SIZE + 1]))+ .await?;+ assert_eq!(+ part_sizes.lock().unwrap().as_slice(),+ &[OBJECT_STORE_PART_SIZE]+ );+ assert_eq!(writer.buffer.content_length(), 1);+ writer.abort().await?;Ok(())}-}-/// Run an async ObjectStore operation.-///-/// Adding a native async API is tracked in <https://github.com/apache/datafusion/issues/23247>-fn block_on_object_store<T>(future: impl Future<Output = Result<T>>) -> Result<T> {- if let Ok(handle) = tokio::runtime::Handle::try_current() {- tokio::task::block_in_place(|| handle.block_on(future))- } else {- exec_err!("No current Tokio runtime available")+ #[tokio::test]+ async fn buffered_tail_does_not_retain_input_allocation() -> Result<()> {+ let tmp_dir = tempdir()?;+ let store: Arc<dyn ObjectStore> =+ Arc::new(LocalFileSystem::new_with_prefix(tmp_dir.path())?);+ let location = Path::from("tail-retention-test");+ let upload = store.put_multipart(&location).await?;+ let dropped = Arc::new(AtomicBool::new(false));+ let data = Bytes::from_owner(DropTrackingOwner {+ data: vec![42; 128],+ dropped: Arc::clone(&dropped),+ });+ let mut writer = ObjectStoreSpillWriter {+ upload,+ buffer: PutPayloadMut::new(),+ size: Arc::new(AtomicU64::new(0)),+ bytes_written: 0,+ };++ writer.write_all(data).await?;+ assert!(+ dropped.load(Ordering::Relaxed),+ "buffered tail should not retain the input allocation"+ );+ writer.abort().await?;+ Ok(())}}diff --git a/datafusion/execution/src/lib.rs b/datafusion/execution/src/lib.rsindex 5af7064f1c..153c0216b8 100644--- a/datafusion/execution/src/lib.rs+++ b/datafusion/execution/src/lib.rs@@ -49,6 +49,6 @@ pub mod registry {pub use async_stream::{Emitter, TryEmitter, async_stream, async_try_stream};pub use disk_manager::DiskManager;pub use registry::FunctionRegistry;-pub use spill_file::{SpillFile, SpillWriter, TempFileFactory};+pub use spill_file::{AsyncSpillWriter, SpillFile, SpillWriter, TempFileFactory};pub use stream::{RecordBatchStream, SendableRecordBatchStream};pub use task::{TaskContext, TaskContextProvider};diff --git a/datafusion/execution/src/spill_file.rs b/datafusion/execution/src/spill_file.rsindex dca5da23f5..e73fbec085 100644--- a/datafusion/execution/src/spill_file.rs+++ b/datafusion/execution/src/spill_file.rs@@ -15,15 +15,18 @@// specific language governing permissions and limitations// under the License.+use async_trait::async_trait;use bytes::Bytes;use datafusion_common::Result;use futures::Stream;+use std::io::Write;use std::path::Path;use std::pin::Pin;use std::sync::Arc;/// Abstraction over a spill file backend./// Implementations handle their own quota enforcement and blocking concerns.+#[async_trait]pub trait SpillFile: Send + Sync {/// Returns the OS path if this is a local file, None otherwise.fn path(&self) -> Option<&Path> {@@ -38,14 +41,93 @@ pub trait SpillFile: Send + Sync {/// Opens a writer for appending data to this file.fn open_writer(&self) -> Result<Box<dyn SpillWriter>>;++ /// Opens an asynchronous writer for appending data to this file.+ ///+ /// The default implementation adapts [`Self::open_writer`] for backwards+ /// compatibility. Backends with native asynchronous I/O should override+ /// this method to avoid blocking an async executor.+ async fn open_async_writer(&self) -> Result<Box<dyn AsyncSpillWriter>> {+ Ok(Box::new(BlockingSpillWriterAdapter {+ inner: self.open_writer()?,+ }))+ }}/// Writer for spill file backends.-pub trait SpillWriter: std::io::Write + Send {+pub trait SpillWriter: Write + Send {/// Intended for close/sync/commit operations.fn finish(&mut self) -> Result<()>;}+/// Asynchronous writer for spill file backends.+///+/// The writer accepts owned [`Bytes`] so asynchronous backends can retain a+/// buffer across an await within [`Self::write_all`] without copying it. Calls+/// are made sequentially and buffers must be persisted in the order received.+/// After `write_all` returns, implementations must not retain the input's+/// backing allocation unless that retained memory is accounted for separately.+/// Backends that buffer a tail for a later call should copy that tail into an+/// allocation sized for the retained bytes.+///+/// DataFusion makes a best-effort attempt to abort writers dropped by query+/// cancellation. Backends should still configure lifecycle cleanup for+/// abandoned uploads because cleanup cannot run after process termination.+#[async_trait]+pub trait AsyncSpillWriter: Send {+ /// Writes all bytes in `data` to the spill file.+ async fn write_all(&mut self, data: Bytes) -> Result<()>;++ /// Flushes buffered data, if supported by the backend.+ ///+ /// This does not finish the writer or require the spill file to become+ /// visible to readers. [`Self::finish`] is the commit boundary.+ async fn flush(&mut self) -> Result<()> {+ Ok(())+ }++ /// Finishes and commits the spill file.+ ///+ /// A successful call is terminal. If committing returns an error, the+ /// caller will invoke [`Self::abort`] before dropping the writer.+ async fn finish(&mut self) -> Result<()>;++ /// Aborts an uncommitted spill write and cleans up backend resources.+ ///+ /// Multipart backends should override this method to explicitly abort an+ /// in-progress upload. A successful call is terminal. DataFusion may retry+ /// an abort that returns an error, including during best-effort drop cleanup.+ ///+ /// Implementations should bound their own abort latency to avoid holding+ /// DataFusion's limited cleanup slots indefinitely. DataFusion does not+ /// impose a timeout because the caller's runtime may not have a time driver.+ async fn abort(&mut self) -> Result<()> {+ Ok(())+ }+}++struct BlockingSpillWriterAdapter {+ inner: Box<dyn SpillWriter>,+}++#[async_trait]+impl AsyncSpillWriter for BlockingSpillWriterAdapter {+ async fn write_all(&mut self, data: Bytes) -> Result<()> {+ self.inner.write_all(&data)?;+ Ok(())+ }++ async fn flush(&mut self) -> Result<()> {+ self.inner.flush()?;+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ self.inner.flush()?;+ self.inner.finish()+ }+}+/// Factory for creating spill files.pub trait TempFileFactory: Send + Sync {fn create_temp_file(&self, description: &str) -> Result<Arc<dyn SpillFile>>;diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 1f927345d2..c16a8de97e 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -88,7 +88,7 @@ num-traits = { workspace = true }parking_lot = { workspace = true }pin-project-lite = { workspace = true }serde_json = { workspace = true, features = ["preserve_order"] }-tokio = { workspace = true }+tokio = { workspace = true, features = ["time"] }[dev-dependencies]arrow-data = { workspace = true }diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rsindex 490ea7cc85..9a149cc7c3 100644--- a/datafusion/physical-plan/src/sorts/sort.rs+++ b/datafusion/physical-plan/src/sorts/sort.rs@@ -421,7 +421,7 @@ impl ExternalSorter {/// Appending globally sorted batches to the in-progress spill file, and clears/// the `globally_sorted_batches` (also its memory reservation) afterwards.- fn consume_and_spill_append(+ async fn consume_and_spill_append(&mut self,globally_sorted_batches: &mut Vec<RecordBatch>,) -> Result<()> {@@ -438,7 +438,9 @@ impl ExternalSorter {debug!("Spilling sort data of ExternalSorter to disk whilst inserting");let batches_to_spill = std::mem::take(globally_sorted_batches);- self.reservation.free();+ // Keep the reservation alive while the batches remain in memory across+ // asynchronous writes. It is released on success or error via RAII.+ let _spill_reservation = self.reservation.take();let (in_progress_file, max_record_batch_size) =self.in_progress_spill_file.as_mut().ok_or_else(|| {@@ -446,7 +448,7 @@ impl ExternalSorter {})?;for batch in batches_to_spill {- let gc_sliced_size = in_progress_file.append_batch(&batch)?;+ let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;*max_record_batch_size = (*max_record_batch_size).max(gc_sliced_size);}@@ -460,12 +462,12 @@ impl ExternalSorter {}/// Finishes the in-progress spill file and moves it to the finished spill files.- fn spill_finish(&mut self) -> Result<()> {+ async fn spill_finish(&mut self) -> Result<()> {let (mut in_progress_file, max_record_batch_memory) =self.in_progress_spill_file.take().ok_or_else(|| {internal_datafusion_err!("Should be called after `spill_append`")})?;- let spill_file = in_progress_file.finish()?;+ let spill_file = in_progress_file.finish_async().await?;if let Some(spill_file) = spill_file {self.finished_spill_files.push(SortedSpillFile {@@ -477,9 +479,26 @@ impl ExternalSorter {Ok(())}+ async fn abort_in_progress_spill(&mut self) {+ if let Some((in_progress_file, _)) = &mut self.in_progress_spill_file+ && let Err(error) = in_progress_file.abort_async().await+ {+ debug!("Failed to abort in-progress sort spill: {error}");+ }+ self.in_progress_spill_file.take();+ }+/// Sorts the in-memory batches and merges them into a single sorted run, then writes/// the result to spill files.async fn sort_and_spill_in_mem_batches(&mut self) -> Result<()> {+ let result = self.try_sort_and_spill_in_mem_batches().await;+ if result.is_err() {+ self.abort_in_progress_spill().await;+ }+ result+ }++ async fn try_sort_and_spill_in_mem_batches(&mut self) -> Result<()> {assert_or_internal_err!(!self.in_mem_batches.is_empty(),"in_mem_batches must not be empty when attempting to sort and spill"@@ -508,14 +527,26 @@ impl ExternalSorter {while let Some(batch) = sorted_stream.next().await {let batch = batch?;- let sorted_size = get_reserved_bytes_for_record_batch(&batch)?;- let reservation_failed = self.reservation.try_grow(sorted_size).is_err();+ // The output is already sorted, so no sorted-copy budget is needed.+ let sorted_size = get_record_batch_memory_size(&batch);+ let spill_workspace = match self.reservation.try_grow(sorted_size) {+ Ok(()) => None,+ Err(_) => {+ let workspace = self.merge_pool.borrow(sorted_size);+ // The batch is already materialized, so account for it while+ // spilling, reusing available workspace before growing the+ // reservation even if this exceeds the pool limit.+ self.reservation.grow(sorted_size - workspace.size());+ Some(workspace)+ }+ };// Even if the reservation is not enough, the batch is already in// memory, so it's okay to combine it with previously sorted// batches, and spill together.globally_sorted_batches.push(batch);- if reservation_failed {- self.consume_and_spill_append(&mut globally_sorted_batches)?; // reservation is freed in spill()+ if let Some(_spill_workspace) = spill_workspace {+ self.consume_and_spill_append(&mut globally_sorted_batches)+ .await?; // reservation is released when the spill completes}}@@ -523,8 +554,9 @@ impl ExternalSorter {// upcoming `self.reserve_memory_for_merge()` may fail due to insufficient memory.drop(sorted_stream);- self.consume_and_spill_append(&mut globally_sorted_batches)?;- self.spill_finish()?;+ self.consume_and_spill_append(&mut globally_sorted_batches)+ .await?;+ self.spill_finish().await?;// Sanity check after spillinglet buffers_cleared_property =@@ -1490,8 +1522,17 @@ impl ExecutionPlan for SortExec {self.schema(),futures::stream::once(async move {while let Some(batch) = input.next().await {- let batch = batch?;- sorter.insert_batch(batch).await?;+ let batch = match batch {+ Ok(batch) => batch,+ Err(error) => {+ sorter.abort_in_progress_spill().await;+ return Err(error);+ }+ };+ if let Err(error) = sorter.insert_batch(batch).await {+ sorter.abort_in_progress_spill().await;+ return Err(error);+ }}drop(input);sorter.sort().await@@ -2115,7 +2156,9 @@ mod proto_tests {#[cfg(test)]mod tests {use std::collections::HashMap;+ use std::path::Path;use std::pin::Pin;+ use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};use std::task::{Context, Poll};use super::*;@@ -2133,22 +2176,106 @@ mod tests {use arrow::array::*;use arrow::compute::SortOptions;use arrow::datatypes::*;+ use async_trait::async_trait;+ use bytes::Bytes;use datafusion_common::ScalarValue;use datafusion_common::cast::as_primitive_array;use datafusion_common::config::ConfigOptions;use datafusion_common::test_util::batches_to_string;use datafusion_execution::RecordBatchStream;use datafusion_execution::config::SessionConfig;+ use datafusion_execution::disk_manager::DiskManagerBuilder;use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool,};use datafusion_execution::runtime_env::RuntimeEnvBuilder;+ use datafusion_execution::{+ AsyncSpillWriter, SpillFile, SpillWriter, TempFileFactory,+ };use datafusion_physical_expr::expressions::{Column, Literal};use datafusion_physical_expr::{DynamicFilterTracking, EquivalenceProperties};use datafusion_physical_expr_common::metrics::MetricValue;use futures::{FutureExt, Stream, TryStreamExt};use insta::assert_snapshot;+ use tokio::sync::Notify;++ struct PendingWriteTempFileFactory {+ write_started: Arc<Notify>,+ aborted: Arc<Notify>,+ abort_count: Arc<AtomicUsize>,+ }++ impl TempFileFactory for PendingWriteTempFileFactory {+ fn create_temp_file(&self, _description: &str) -> Result<Arc<dyn SpillFile>> {+ Ok(Arc::new(PendingWriteSpillFile {+ write_started: Arc::clone(&self.write_started),+ aborted: Arc::clone(&self.aborted),+ abort_count: Arc::clone(&self.abort_count),+ }))+ }+ }++ struct PendingWriteSpillFile {+ write_started: Arc<Notify>,+ aborted: Arc<Notify>,+ abort_count: Arc<AtomicUsize>,+ }++ #[async_trait]+ impl SpillFile for PendingWriteSpillFile {+ fn path(&self) -> Option<&Path> {+ None+ }++ fn size(&self) -> Option<u64> {+ Some(0)+ }++ fn read_stream(+ &self,+ ) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes>> + Send>>> {+ Ok(Box::pin(futures::stream::empty()))+ }++ fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {+ datafusion_common::not_impl_err!(+ "test backend only supports asynchronous writes"+ )+ }++ async fn open_async_writer(&self) -> Result<Box<dyn AsyncSpillWriter>> {+ Ok(Box::new(PendingWriteWriter {+ write_started: Arc::clone(&self.write_started),+ aborted: Arc::clone(&self.aborted),+ abort_count: Arc::clone(&self.abort_count),+ }))+ }+ }++ struct PendingWriteWriter {+ write_started: Arc<Notify>,+ aborted: Arc<Notify>,+ abort_count: Arc<AtomicUsize>,+ }++ #[async_trait]+ impl AsyncSpillWriter for PendingWriteWriter {+ async fn write_all(&mut self, _data: Bytes) -> Result<()> {+ self.write_started.notify_one();+ std::future::pending().await+ }++ async fn finish(&mut self) -> Result<()> {+ Ok(())+ }++ async fn abort(&mut self) -> Result<()> {+ self.abort_count.fetch_add(1, AtomicOrdering::Relaxed);+ self.aborted.notify_one();+ Ok(())+ }+ }#[derive(Debug, Clone)]pub struct SortedUnboundedExec {@@ -3831,6 +3958,89 @@ mod tests {Ok(())}+ #[rstest::rstest]+ #[case::no_workspace(0)]+ #[case::partial_workspace(6)]+ #[case::full_workspace(12)]+ #[tokio::test]+ async fn test_spill_reservation_held_during_async_write(+ #[case] workspace_bytes: usize,+ ) -> Result<()> {+ let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(0));+ let write_started = Arc::new(Notify::new());+ let aborted = Arc::new(Notify::new());+ let abort_count = Arc::new(AtomicUsize::new(0));+ let disk_manager_builder = DiskManagerBuilder::default().with_temp_file_factory(+ Arc::new(PendingWriteTempFileFactory {+ write_started: Arc::clone(&write_started),+ aborted: Arc::clone(&aborted),+ abort_count: Arc::clone(&abort_count),+ }),+ );+ let runtime = RuntimeEnvBuilder::new()+ .with_memory_pool(Arc::clone(&pool))+ .with_disk_manager_builder(disk_manager_builder)+ .build_arc()?;+ let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));+ let metrics = ExecutionPlanMetricsSet::new();+ let mut sorter = ExternalSorter::new(+ 0,+ Arc::clone(&schema),+ [PhysicalSortExpr::new_default(Arc::new(Column::new("x", 0)))].into(),+ 128,+ 0,+ usize::MAX,+ SpillCompression::Uncompressed,+ &metrics,+ runtime,+ )?;+ let batch = RecordBatch::try_new(+ schema,+ vec![Arc::new(Int32Array::from(vec![3, 2, 1]))],+ )?;+ let reserved_bytes = get_record_batch_memory_size(&batch);+ sorter+ .reservation+ .grow(get_reserved_bytes_for_record_batch(&batch)?);+ sorter.merge_reservation.grow(workspace_bytes);+ sorter.merge_pool.retain(workspace_bytes);+ let merge_pool = Arc::clone(&sorter.merge_pool);+ sorter.in_mem_batches.push(batch);++ #[expect(clippy::disallowed_methods)] // spawn allowed only in tests+ let task =+ tokio::spawn(async move { sorter.sort_and_spill_in_mem_batches().await });++ if tokio::time::timeout(+ std::time::Duration::from_secs(5),+ write_started.notified(),+ )+ .await+ .is_err()+ {+ task.abort();+ let _ = task.await;+ panic!("spill write did not start before the timeout");+ }+ // An idle retained workspace must not mask an early release of the loan.+ merge_pool.release_unused();+ assert_eq!(merge_pool.reserved(), workspace_bytes);+ assert_eq!(+ pool.reserved(),+ reserved_bytes,+ "resident batches must remain accounted for while an async spill is pending"+ );++ task.abort();+ assert!(task.await.unwrap_err().is_cancelled());+ tokio::time::timeout(std::time::Duration::from_secs(1), aborted.notified())+ .await+ .expect("cancelling the spill should abort its writer");+ assert_eq!(abort_count.load(AtomicOrdering::Relaxed), 1);+ assert_eq!(pool.reserved(), 0);+ Ok(())+ }+/// Verifies that `ExternalSorter::sort()` transfers the pre-reserved/// merge bytes to the merge stream via `take()`, rather than leaving/// them in the sorter (via `new_empty()`).diff --git a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs b/datafusion/physical-plan/src/spill/in_progress_spill_file.rsindex 71d7cce1bc..2f4ddd3398 100644--- a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs+++ b/datafusion/physical-plan/src/spill/in_progress_spill_file.rs@@ -17,7 +17,7 @@//! Define the `InProgressSpillFile` struct, which represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`.-use datafusion_common::Result;+use datafusion_common::{Result, internal_datafusion_err};use std::sync::Arc;use arrow::array::RecordBatch;@@ -25,28 +25,33 @@ use datafusion_common::exec_datafusion_err;use datafusion_execution::spill_file::SpillFile;use super::{- IPCStreamWriter, gc_view_arrays,+ AsyncIPCStreamWriter, IPCStreamEncoder, IPCStreamWriter, gc_view_arrays,spill_manager::{GetSlicedSize, SpillManager},};+enum InProgressWriter {+ Sync(IPCStreamWriter),+ Async(AsyncIPCStreamWriter),+}+/// Represents an in-progress spill file used for writing `RecordBatch`es to disk, created by `SpillManager`./// Caller is able to use this struct to incrementally append in-memory batches to/// the file, and then finalize the file by calling the `finish` method.pub struct InProgressSpillFile {- pub(crate) spill_writer: Arc<SpillManager>,+ pub(crate) spill_manager: Arc<SpillManager>,/// Lazily initialized writer- writer: Option<IPCStreamWriter>,+ writer: Option<InProgressWriter>,/// Lazily initialized in-progress file, it will be moved out when the `finish` method is invokedin_progress_file: Option<Arc<dyn SpillFile>>,}impl InProgressSpillFile {pub fn new(- spill_writer: Arc<SpillManager>,+ spill_manager: Arc<SpillManager>,in_progress_file: Arc<dyn SpillFile>,) -> Self {Self {- spill_writer,+ spill_manager,in_progress_file: Some(in_progress_file),writer: None,}@@ -78,35 +83,92 @@ impl InProgressSpillFile {// Individual batches may have different schemas (e.g., different nullability)// when they come from different branches of a UnionExec. The SpillManager's// schema represents the canonical schema that all batches should conform to.- let schema = self.spill_writer.schema();+ let schema = self.spill_manager.schema();if let Some(in_progress_file) = &self.in_progress_file {let spill_writer = in_progress_file.open_writer()?;- self.writer = Some(IPCStreamWriter::new(+ self.writer = Some(InProgressWriter::Sync(IPCStreamWriter::new(spill_writer,schema.as_ref(),- self.spill_writer.compression,- )?);+ self.spill_manager.compression,+ )?));// Update metrics- self.spill_writer.metrics.spill_file_count.add(1);- let header_bytes = self.writer.as_ref().unwrap().bytes_written();- self.spill_writer.metrics.spilled_bytes.add(header_bytes);+ self.spill_manager.metrics.spill_file_count.add(1);}}- if let Some(writer) = &mut self.writer {+ if let Some(InProgressWriter::Sync(writer)) = &mut self.writer {// The writer calculates how many serialized bytes were emittedlet (spilled_rows, delta_bytes) = writer.write(&gc_batch)?;- self.spill_writer.metrics.spilled_rows.add(spilled_rows);- self.spill_writer.metrics.spilled_bytes.add(delta_bytes);+ self.spill_manager.metrics.spilled_rows.add(spilled_rows);+ self.spill_manager.metrics.spilled_bytes.add(delta_bytes);+ } else if self.writer.is_some() {+ return Err(exec_datafusion_err!(+ "Cannot use synchronous append after asynchronous spill writing has started"+ ));+ }+ gc_batch.get_sliced_size()+ }++ /// Appends a `RecordBatch` using the asynchronous spill writer.+ pub async fn append_batch_async(&mut self, batch: &RecordBatch) -> Result<usize> {+ if self.in_progress_file.is_none() {+ return Err(exec_datafusion_err!(+ "Append operation failed: No active in-progress file. The file may have already been finalized."+ ));+ }++ let gc_batch = gc_view_arrays(batch)?;++ if self.writer.is_none() {+ let schema = self.spill_manager.schema();+ if let Some(in_progress_file) = &self.in_progress_file {+ // Validate the IPC schema and compression options before opening+ // a remote upload that would otherwise need to be aborted.+ let encoder = IPCStreamEncoder::new(+ schema.as_ref(),+ self.spill_manager.compression,+ )?;+ let spill_writer = in_progress_file.open_async_writer().await?;++ self.writer = Some(InProgressWriter::Async(AsyncIPCStreamWriter::new(+ spill_writer,+ encoder,+ )));++ self.spill_manager.metrics.spill_file_count.add(1);+ }+ }++ match &mut self.writer {+ Some(InProgressWriter::Async(writer)) => {+ let (spilled_rows, delta_bytes) = writer.write(&gc_batch).await?;++ self.spill_manager.metrics.spilled_rows.add(spilled_rows);+ self.spill_manager.metrics.spilled_bytes.add(delta_bytes);+ }+ Some(InProgressWriter::Sync(_)) => {+ return Err(exec_datafusion_err!(+ "Cannot use asynchronous append after synchronous spill writing has started"+ ));+ }+ None => {+ return Err(internal_datafusion_err!(+ "Asynchronous spill writer was not initialized"+ ));+ }}gc_batch.get_sliced_size()}pub fn flush(&mut self) -> Result<()> {- if let Some(writer) = &mut self.writer {+ if let Some(InProgressWriter::Sync(writer)) = &mut self.writer {writer.flush()?;+ } else if self.writer.is_some() {+ return Err(exec_datafusion_err!(+ "Cannot use synchronous flush for an asynchronous spill writer"+ ));}Ok(())}@@ -125,16 +187,62 @@ impl InProgressSpillFile {"Finish operation failed: file has already been finalized."));}- if let Some(mut writer) = self.writer.take() {+ if matches!(self.writer, Some(InProgressWriter::Async(_))) {+ return Err(exec_datafusion_err!(+ "Cannot use synchronous finish for an asynchronous spill writer"+ ));+ }+ if let Some(InProgressWriter::Sync(mut writer)) = self.writer.take() {// Finish the writer and capture any final trailing bytes emittedlet delta_bytes = writer.finish()?;- self.spill_writer.metrics.spilled_bytes.add(delta_bytes);+ self.spill_manager.metrics.spilled_bytes.add(delta_bytes);+ } else {+ return Ok(None);+ }++ Ok(self.in_progress_file.take())+ }++ /// Finalizes an asynchronous spill write, returning the completed file.+ pub async fn finish_async(&mut self) -> Result<Option<Arc<dyn SpillFile>>> {+ if self.in_progress_file.is_none() && self.writer.is_none() {+ return Err(exec_datafusion_err!(+ "Finish operation failed: file has already been finalized."+ ));+ }+ if matches!(self.writer, Some(InProgressWriter::Sync(_))) {+ return Err(exec_datafusion_err!(+ "Cannot use asynchronous finish for a synchronous spill writer"+ ));+ }+ if let Some(InProgressWriter::Async(writer)) = &mut self.writer {+ let delta_bytes = writer.finish().await?;+ self.spill_manager.metrics.spilled_bytes.add(delta_bytes);} else {return Ok(None);}+ self.writer.take();Ok(self.in_progress_file.take())}++ /// Aborts an asynchronous spill write and discards its file.+ pub async fn abort_async(&mut self) -> Result<()> {+ if matches!(self.writer, Some(InProgressWriter::Sync(_))) {+ return Err(exec_datafusion_err!(+ "Cannot use asynchronous abort for a synchronous spill writer"+ ));+ }++ let result = if let Some(InProgressWriter::Async(writer)) = &mut self.writer {+ writer.abort().await+ } else {+ Ok(())+ };+ self.writer.take();+ self.in_progress_file.take();+ result+ }}#[cfg(test)]diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex e3c4b2cf30..3f56d2a6e0 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -21,7 +21,7 @@ pub(crate) mod in_progress_spill_file;pub(crate) mod replayable_spill_input;pub(crate) mod spill_manager;pub mod spill_pool;-use datafusion_execution::spill_file::SpillWriter;+use datafusion_execution::spill_file::{AsyncSpillWriter, SpillWriter};// Moved for refactor, re-export to keep the public API stablepub use datafusion_common::utils::memory::get_record_batch_memory_size;// Re-export SpillManager for doctests only (hidden from public docs)@@ -30,7 +30,7 @@ pub use spill_manager::SpillManager;use std::collections::VecDeque;use std::pin::Pin;-use std::sync::Arc;+use std::sync::{Arc, LazyLock};use std::task::{Context, Poll};use arrow::array::{@@ -43,14 +43,16 @@ use arrow::datatypes::{ByteViewType, Schema, SchemaRef};use arrow::ipc::{MetadataVersion,reader::StreamDecoder,- writer::{IpcWriteOptions, StreamWriter},+ writer::{IpcWriteOptions, StreamEncoder},};use arrow::record_batch::RecordBatch;use arrow_data::ArrayDataBuilder;+#[cfg(test)]+use arrow_ipc::writer::StreamWriter;use arrow_ipc::{CompressionType, root_as_message};-use datafusion_common::Result;use datafusion_common::config::SpillCompression;+use datafusion_common::{Result, internal_datafusion_err};use datafusion_execution::RecordBatchStream;use datafusion_execution::spill_file::SpillFile;use futures::Stream;@@ -464,59 +466,205 @@ impl RecordBatchStream for SpillReaderStream {}}-/// A wrapper that counts the exact compressed IPC bytes written by Arrow.-///-/// Arrow's `StreamWriter` does not return the number of bytes written during its-/// `write()` calls. To accurately track the `spilled_bytes` metrics (especially-/// when LZ4/ZSTD compression is applied), we must intercept the `std::io::Write`-/// trait boundary to count the final serialized payload size.-pub(crate) struct TrackingSpillWriter {- inner: Box<dyn SpillWriter>,- pub(crate) total_bytes_written: usize,+/// Write in Arrow IPC Stream format to an underlying `SpillWriter` backend.+/// Stream format also supports dictionary replacement.+struct IPCStreamWriter {+ writer: Box<dyn SpillWriter>,+ encoder: IPCStreamEncoder,}-impl TrackingSpillWriter {- pub fn new(inner: Box<dyn SpillWriter>) -> Self {- Self {- inner,- total_bytes_written: 0,+impl IPCStreamWriter {+ pub fn new(+ spill_writer: Box<dyn SpillWriter>,+ schema: &Schema,+ spill_compression: SpillCompression,+ ) -> Result<Self> {+ Ok(Self {+ writer: spill_writer,+ encoder: IPCStreamEncoder::new(schema, spill_compression)?,+ })+ }++ pub fn write(&mut self, batch: &RecordBatch) -> Result<(usize, usize)> {+ use std::io::Write;+ let (rows, bytes, buffers) = self.encoder.encode(batch)?;+ for buffer in buffers {+ self.writer.write_all(buffer.as_slice())?;}+ Ok((rows, bytes))}- pub fn finish(mut self) -> Result<()> {- self.inner.finish()+ pub fn flush(&mut self) -> Result<()> {+ use std::io::Write;+ self.writer.flush()?;+ Ok(())+ }++ pub fn finish(&mut self) -> Result<usize> {+ use std::io::Write;+ let (bytes, buffers) = self.encoder.finish()?;+ for buffer in buffers {+ self.writer.write_all(buffer.as_slice())?;+ }+ self.writer.flush()?;+ self.writer.finish()?;+ Ok(bytes)}}-impl std::io::Write for TrackingSpillWriter {- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {- let n = self.inner.write(buf)?;+/// Writes Arrow IPC Stream data through an [`AsyncSpillWriter`] interface.+struct AsyncIPCStreamWriter {+ writer: Option<Box<dyn AsyncSpillWriter>>,+ encoder: IPCStreamEncoder,+ abort_handle: Option<tokio::runtime::Handle>,+}++/// Bounds the number of detached cleanup tasks after query cancellation. If all+/// permits are occupied, the writer is dropped and the backend lifecycle policy+/// is the remaining cleanup mechanism.+const MAX_CONCURRENT_SPILL_ABORTS: usize = 8;+static SPILL_ABORT_PERMITS: LazyLock<Arc<tokio::sync::Semaphore>> =+ LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_SPILL_ABORTS)));++impl AsyncIPCStreamWriter {+ pub fn new(+ spill_writer: Box<dyn AsyncSpillWriter>,+ encoder: IPCStreamEncoder,+ ) -> Self {+ Self {+ writer: Some(spill_writer),+ encoder,+ abort_handle: tokio::runtime::Handle::try_current().ok(),+ }+ }++ pub async fn write(&mut self, batch: &RecordBatch) -> Result<(usize, usize)> {+ let (rows, bytes, buffers) = match self.encoder.encode(batch) {+ Ok(encoded) => encoded,+ Err(error) => {+ self.abort_after_error().await;+ return Err(error);+ }+ };+ for buffer in buffers {+ let result = self+ .writer+ .as_mut()+ .ok_or_else(|| {+ internal_datafusion_err!("Spill writer is no longer active")+ })?+ .write_all(buffer_to_bytes(buffer))+ .await;+ if let Err(error) = result {+ self.abort_after_error().await;+ return Err(error);+ }+ }+ Ok((rows, bytes))+ }++ pub async fn finish(&mut self) -> Result<usize> {+ let (bytes, buffers) = match self.encoder.finish() {+ Ok(encoded) => encoded,+ Err(error) => {+ self.abort_after_error().await;+ return Err(error);+ }+ };+ for buffer in buffers {+ let result = self+ .writer+ .as_mut()+ .ok_or_else(|| {+ internal_datafusion_err!("Spill writer is no longer active")+ })?+ .write_all(buffer_to_bytes(buffer))+ .await;+ if let Err(error) = result {+ self.abort_after_error().await;+ return Err(error);+ }+ }+ let result = self+ .writer+ .as_mut()+ .ok_or_else(|| internal_datafusion_err!("Spill writer is no longer active"))?+ .flush()+ .await;+ if let Err(error) = result {+ self.abort_after_error().await;+ return Err(error);+ }+ let result = self+ .writer+ .as_mut()+ .ok_or_else(|| internal_datafusion_err!("Spill writer is no longer active"))?+ .finish()+ .await;+ if let Err(error) = result {+ self.abort_after_error().await;+ return Err(error);+ }+ self.writer.take();+ Ok(bytes)+ }- self.total_bytes_written += n;+ pub async fn abort(&mut self) -> Result<()> {+ let Some(writer) = self.writer.as_mut() else {+ return Ok(());+ };+ writer.abort().await?;+ self.writer = None;+ Ok(())+ }- Ok(n)+ async fn abort_after_error(&mut self) {+ if let Err(error) = self.abort().await {+ debug!("Failed to abort spill writer: {error}");+ }}+}- fn flush(&mut self) -> std::io::Result<()> {- self.inner.flush()+impl Drop for AsyncIPCStreamWriter {+ fn drop(&mut self) {+ let Some(mut writer) = self.writer.take() else {+ return;+ };++ let Some(handle) = tokio::runtime::Handle::try_current()+ .ok()+ .or_else(|| self.abort_handle.take())+ else {+ debug!("Unable to abort dropped spill writer without a Tokio runtime");+ return;+ };++ let Ok(permit) = Arc::clone(&SPILL_ABORT_PERMITS).try_acquire_owned() else {+ debug!(+ "Skipping spill writer abort because the cleanup concurrency limit was reached"+ );+ return;+ };++ let _abort_task = handle.spawn(async move {+ // Backends bound their own abort latency; the runtime may not have+ // a time driver. See the AsyncSpillWriter::abort contract.+ if let Err(error) = writer.abort().await {+ debug!("Failed to abort dropped spill writer: {error}");+ }+ // Keep the permit until cleanup finishes.+ drop(permit);+ });}}-/// Write in Arrow IPC Stream format to an underlying `SpillWriter` backend.-/// Stream format also supports dictionary replacement.-struct IPCStreamWriter {- /// Inner writer- writer: Option<StreamWriter<TrackingSpillWriter>>,- /// Batches written- num_batches: usize,- /// Rows written- num_rows: usize,- /// Bytes written- num_bytes: usize,+/// Encodes Arrow IPC streams without performing I/O.+struct IPCStreamEncoder {+ encoder: Option<StreamEncoder>,}-impl IPCStreamWriter {- /// Create new writer+impl IPCStreamEncoder {+ /// Create a new encoder.////// # Codec contract///@@ -528,11 +676,7 @@ impl IPCStreamWriter {/// contract local and build-visible during Cargo feature resolution,/// rather than relying solely on workspace-level feature unification;/// see #21917.- pub fn new(- spill_writer: Box<dyn SpillWriter>,- schema: &Schema,- spill_compression: SpillCompression,- ) -> Result<Self> {+ fn new(schema: &Schema, spill_compression: SpillCompression) -> Result<Self> {let metadata_version = MetadataVersion::V5;// Depending on the schema, some array types such as StringViewArray require larger (16 byte in this case) alignment.// If the actual buffer layout after IPC read does not satisfy the alignment requirement,@@ -546,68 +690,37 @@ impl IPCStreamWriter {let compression_type = Option::<CompressionType>::from(spill_compression);write_options = write_options.try_with_compression(compression_type)?;- let adapter = TrackingSpillWriter::new(spill_writer);- let writer = StreamWriter::try_new_with_options(adapter, schema, write_options)?;-+ let encoder = StreamEncoder::try_new_with_options(schema, write_options)?;Ok(Self {- num_batches: 0,- num_rows: 0,- num_bytes: 0,- writer: Some(writer),+ encoder: Some(encoder),})}- /// Writes a single batch to the IPC stream and updates the internal counters.- ///- /// Returns a tuple containing the change in the number of rows and bytes written.- pub fn write(&mut self, batch: &RecordBatch) -> Result<(usize, usize)> {- let writer = self.writer.as_mut().unwrap();-- let bytes_before = writer.get_ref().total_bytes_written;- writer.write(batch)?;- let bytes_after = writer.get_ref().total_bytes_written;- self.num_batches += 1;- let delta_num_rows = batch.num_rows();- self.num_rows += delta_num_rows;- let delta_num_bytes = bytes_after - bytes_before;- self.num_bytes += delta_num_bytes;- Ok((delta_num_rows, delta_num_bytes))+ fn encode(&mut self, batch: &RecordBatch) -> Result<(usize, usize, Vec<Buffer>)> {+ let buffers = self.encoder.as_mut().unwrap().encode(batch)?;+ let bytes = buffers.iter().map(Buffer::len).sum();+ Ok((batch.num_rows(), bytes, buffers))}- pub fn flush(&mut self) -> Result<()> {- use std::io::Write;- if let Some(writer) = &mut self.writer {- writer.get_mut().flush()?;- }- Ok(())+ fn finish(&mut self) -> Result<(usize, Vec<Buffer>)> {+ let buffers = self.encoder.take().unwrap().finish()?;+ let bytes = buffers.iter().map(Buffer::len).sum();+ Ok((bytes, buffers))}+}- /// Finish the writer.- ///- /// Returns the number of trailing bytes written during the finish operation- /// (e.g., IPC metadata and footers).- pub fn finish(&mut self) -> Result<usize> {- let mut writer = self.writer.take().unwrap();-- let bytes_before = writer.get_ref().total_bytes_written;- writer.finish()?; // Writes IPC tail-- // Extract the adapter and flush the final bytes- let adapter = writer.into_inner()?;- let bytes_after = adapter.total_bytes_written;- adapter.finish()?;+struct ArrowBufferOwner(Buffer);- Ok(bytes_after - bytes_before)- }- /// Returns the total number of bytes written so far- pub fn bytes_written(&self) -> usize {- self.writer- .as_ref()- .map(|w| w.get_ref().total_bytes_written)- .unwrap_or(0)+impl AsRef<[u8]> for ArrowBufferOwner {+ fn as_ref(&self) -> &[u8] {+ self.0.as_slice()}}+fn buffer_to_bytes(buffer: Buffer) -> bytes::Bytes {+ bytes::Bytes::from_owner(ArrowBufferOwner(buffer))+}+// Returns the maximum byte alignment required by any field in the schema (>= 8), derived from Arrow buffer layouts.fn get_max_alignment_for_schema(schema: &Schema) -> usize {let minimum_alignment = 8;@@ -811,8 +924,309 @@ mod tests {use arrow::array::{ArrayRef, Int32Array, StringArray};use arrow::compute::cast;use arrow::datatypes::{DataType, Field};+ use arrow::ipc::reader::StreamReader;use datafusion_execution::runtime_env::RuntimeEnv;use futures::StreamExt as _;+ use std::io::Cursor;+ use std::sync::Mutex;+ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+ use std::time::Duration;+ use tokio::sync::Notify;++ struct YieldingAsyncWriter {+ data: Arc<Mutex<Vec<u8>>>,+ writes: Arc<AtomicUsize>,+ finished: Arc<AtomicBool>,+ aborted: Arc<AtomicBool>,+ }++ struct FailingAsyncWriter {+ aborted: Arc<AtomicBool>,+ }++ struct PendingFinishAsyncWriter {+ finish_started: Arc<Notify>,+ aborted: Arc<Notify>,+ abort_count: Arc<AtomicUsize>,+ }++ struct FailOnceAbortWriter {+ abort_count: Arc<AtomicUsize>,+ cleanup_done: Arc<Notify>,+ }++ #[async_trait::async_trait]+ impl AsyncSpillWriter for FailingAsyncWriter {+ async fn write_all(&mut self, _data: bytes::Bytes) -> Result<()> {+ datafusion_common::exec_err!("injected spill write failure")+ }++ async fn finish(&mut self) -> Result<()> {+ Ok(())+ }++ async fn abort(&mut self) -> Result<()> {+ self.aborted.store(true, Ordering::Relaxed);+ Ok(())+ }+ }++ #[async_trait::async_trait]+ impl AsyncSpillWriter for YieldingAsyncWriter {+ async fn write_all(&mut self, data: bytes::Bytes) -> Result<()> {+ tokio::task::yield_now().await;+ self.writes.fetch_add(1, Ordering::Relaxed);+ self.data.lock().unwrap().extend_from_slice(&data);+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ self.finished.store(true, Ordering::Relaxed);+ Ok(())+ }++ async fn abort(&mut self) -> Result<()> {+ self.aborted.store(true, Ordering::Relaxed);+ Ok(())+ }+ }++ #[async_trait::async_trait]+ impl AsyncSpillWriter for PendingFinishAsyncWriter {+ async fn write_all(&mut self, _data: bytes::Bytes) -> Result<()> {+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ self.finish_started.notify_one();+ std::future::pending().await+ }++ async fn abort(&mut self) -> Result<()> {+ self.abort_count.fetch_add(1, Ordering::Relaxed);+ self.aborted.notify_one();+ Ok(())+ }+ }++ #[async_trait::async_trait]+ impl AsyncSpillWriter for FailOnceAbortWriter {+ async fn write_all(&mut self, _data: bytes::Bytes) -> Result<()> {+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ Ok(())+ }++ async fn abort(&mut self) -> Result<()> {+ if self.abort_count.fetch_add(1, Ordering::Relaxed) == 0 {+ return datafusion_common::exec_err!("injected spill abort failure");+ }+ self.cleanup_done.notify_one();+ Ok(())+ }+ }++ #[tokio::test]+ async fn test_async_ipc_stream_writer() -> Result<()> {+ let batch1 = build_table_i32(+ ("a", &vec![0, 1, 2]),+ ("b", &vec![3, 4, 5]),+ ("c", &vec![6, 7, 8]),+ );+ let batch2 = build_table_i32(+ ("a", &vec![9, 10, 11]),+ ("b", &vec![12, 13, 14]),+ ("c", &vec![15, 16, 17]),+ );+ let schema = batch1.schema();+ let data = Arc::new(Mutex::new(Vec::new()));+ let writes = Arc::new(AtomicUsize::new(0));+ let finished = Arc::new(AtomicBool::new(false));+ let aborted = Arc::new(AtomicBool::new(false));++ let spill_writer = YieldingAsyncWriter {+ data: Arc::clone(&data),+ writes: Arc::clone(&writes),+ finished: Arc::clone(&finished),+ aborted: Arc::clone(&aborted),+ };+ let encoder =+ IPCStreamEncoder::new(schema.as_ref(), SpillCompression::Uncompressed)?;+ let mut writer = AsyncIPCStreamWriter::new(Box::new(spill_writer), encoder);++ let (_, first_bytes) = writer.write(&batch1).await?;+ let (_, second_bytes) = writer.write(&batch2).await?;+ let trailing_bytes = writer.finish().await?;++ assert!(finished.load(Ordering::Relaxed));+ assert!(writes.load(Ordering::Relaxed) > 2);+ drop(writer);+ assert!(!aborted.load(Ordering::Relaxed));++ let encoded = data.lock().unwrap().clone();+ assert_eq!(encoded.len(), first_bytes + second_bytes + trailing_bytes);+ let decoded = StreamReader::try_new(Cursor::new(encoded), None)?+ .collect::<arrow::error::Result<Vec<_>>>()?;+ assert_eq!(decoded, vec![batch1, batch2]);+ Ok(())+ }++ #[test]+ fn test_buffer_to_bytes_is_zero_copy() {+ let buffer = Buffer::from(vec![0_u8, 1, 2, 3]).slice_with_length(1, 2);+ let expected_ptr = buffer.as_ptr();+ let bytes = buffer_to_bytes(buffer);++ assert_eq!(bytes.as_ptr(), expected_ptr);+ assert_eq!(bytes.as_ref(), &[1, 2]);+ }++ #[tokio::test]+ async fn test_async_ipc_stream_writer_aborts_after_write_error() -> Result<()> {+ let batch = build_table_i32(+ ("a", &vec![0, 1, 2]),+ ("b", &vec![3, 4, 5]),+ ("c", &vec![6, 7, 8]),+ );+ let aborted = Arc::new(AtomicBool::new(false));+ let encoder = IPCStreamEncoder::new(+ batch.schema().as_ref(),+ SpillCompression::Uncompressed,+ )?;+ let mut writer = AsyncIPCStreamWriter::new(+ Box::new(FailingAsyncWriter {+ aborted: Arc::clone(&aborted),+ }),+ encoder,+ );++ let error = writer.write(&batch).await.unwrap_err();+ assert!(error.to_string().contains("injected spill write failure"));+ assert!(aborted.load(Ordering::Relaxed));+ Ok(())+ }++ #[tokio::test]+ async fn test_async_ipc_stream_writer_aborts_cancelled_finish() -> Result<()> {+ let batch = build_table_i32(+ ("a", &vec![0, 1, 2]),+ ("b", &vec![3, 4, 5]),+ ("c", &vec![6, 7, 8]),+ );+ let finish_started = Arc::new(Notify::new());+ let aborted = Arc::new(Notify::new());+ let abort_count = Arc::new(AtomicUsize::new(0));++ #[expect(clippy::disallowed_methods)] // spawn allowed only in tests+ let task = tokio::spawn({+ let finish_started = Arc::clone(&finish_started);+ let aborted = Arc::clone(&aborted);+ let abort_count = Arc::clone(&abort_count);+ async move {+ let encoder = IPCStreamEncoder::new(+ batch.schema().as_ref(),+ SpillCompression::Uncompressed,+ )?;+ let mut writer = AsyncIPCStreamWriter::new(+ Box::new(PendingFinishAsyncWriter {+ finish_started,+ aborted,+ abort_count,+ }),+ encoder,+ );+ writer.write(&batch).await?;+ writer.finish().await?;+ Result::<()>::Ok(())+ }+ });++ if tokio::time::timeout(Duration::from_secs(5), finish_started.notified())+ .await+ .is_err()+ {+ task.abort();+ let _ = task.await;+ panic!("spill finish did not start before the timeout");+ }+ task.abort();+ assert!(task.await.unwrap_err().is_cancelled());+ tokio::time::timeout(Duration::from_secs(1), aborted.notified())+ .await+ .expect("dropping a cancelled writer should abort its upload");+ assert_eq!(abort_count.load(Ordering::Relaxed), 1);+ Ok(())+ }++ #[tokio::test]+ async fn test_async_ipc_stream_writer_retries_failed_abort_on_drop() -> Result<()> {+ let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);+ let abort_count = Arc::new(AtomicUsize::new(0));+ let cleanup_done = Arc::new(Notify::new());+ let encoder = IPCStreamEncoder::new(&schema, SpillCompression::Uncompressed)?;+ let mut writer = AsyncIPCStreamWriter::new(+ Box::new(FailOnceAbortWriter {+ abort_count: Arc::clone(&abort_count),+ cleanup_done: Arc::clone(&cleanup_done),+ }),+ encoder,+ );++ assert!(writer.abort().await.is_err());+ assert_eq!(abort_count.load(Ordering::Relaxed), 1);+ drop(writer);++ tokio::time::timeout(Duration::from_secs(1), cleanup_done.notified())+ .await+ .expect("dropping a writer should retry an abort that returned an error");+ assert_eq!(abort_count.load(Ordering::Relaxed), 2);+ Ok(())+ }++ #[test]+ fn test_async_ipc_stream_writer_aborts_on_current_runtime() -> Result<()> {+ let batch = build_table_i32(+ ("a", &vec![0, 1, 2]),+ ("b", &vec![3, 4, 5]),+ ("c", &vec![6, 7, 8]),+ );+ let aborted = Arc::new(AtomicBool::new(false));+ let first_runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?;+ let writer = {+ let _runtime_guard = first_runtime.enter();+ let spill_writer = YieldingAsyncWriter {+ data: Arc::new(Mutex::new(Vec::new())),+ writes: Arc::new(AtomicUsize::new(0)),+ finished: Arc::new(AtomicBool::new(false)),+ aborted: Arc::clone(&aborted),+ };+ let encoder = IPCStreamEncoder::new(+ batch.schema().as_ref(),+ SpillCompression::Uncompressed,+ )?;+ AsyncIPCStreamWriter::new(Box::new(spill_writer), encoder)+ };+ drop(first_runtime);++ let second_runtime = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?;+ second_runtime.block_on(async {+ drop(writer);+ tokio::time::timeout(Duration::from_secs(1), async {+ while !aborted.load(Ordering::Relaxed) {+ tokio::task::yield_now().await;+ }+ })+ .await+ .expect("drop should abort on the current live runtime");+ });+ Ok(())+ }#[tokio::test]async fn test_batch_spill_and_read() -> Result<()> {@@ -1444,14 +1858,14 @@ mod tests {expected_spilled_rows: usize,) -> Result<()> {let actual_spill_file_count = in_progress_file- .spill_writer+ .spill_manager.metrics.spill_file_count.value();let actual_spilled_bytes =- in_progress_file.spill_writer.metrics.spilled_bytes.value();+ in_progress_file.spill_manager.metrics.spilled_bytes.value();let actual_spilled_rows =- in_progress_file.spill_writer.metrics.spilled_rows.value();+ in_progress_file.spill_manager.metrics.spilled_rows.value();assert_eq!(actual_spill_file_count, expected_spill_file_count,diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex aee9e917c7..1db4f53982 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -27,6 +27,7 @@ use datafusion_common::{DataFusionError, Result, config::SpillCompression};use datafusion_execution::SendableRecordBatchStream;use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_execution::spill_file::SpillFile;+use log::debug;use std::borrow::Borrow;use std::sync::Arc;@@ -150,18 +151,29 @@ impl SpillManager {let mut in_progress_file = self.create_in_progress_file(request_description)?;- let mut max_record_batch_size = 0;+ let result = async {+ let mut max_record_batch_size = 0;- while let Some(batch) = stream.next().await {- let batch = batch?;- let gc_sliced_size = in_progress_file.append_batch(&batch)?;+ while let Some(batch) = stream.next().await {+ let batch = batch?;+ let gc_sliced_size = in_progress_file.append_batch_async(&batch).await?;- max_record_batch_size = max_record_batch_size.max(gc_sliced_size);+ max_record_batch_size = max_record_batch_size.max(gc_sliced_size);+ }++ let file = in_progress_file.finish_async().await?;++ Ok(file.map(|f| (f, max_record_batch_size)))}+ .await;- let file = in_progress_file.finish()?;+ if result.is_err()+ && let Err(error) = in_progress_file.abort_async().await+ {+ debug!("Failed to abort in-progress stream spill: {error}");+ }- Ok(file.map(|f| (f, max_record_batch_size)))+ result}/// Reads a spill file as a stream. The file must be created by the current@@ -255,14 +267,89 @@ mod tests {use crate::common::collect;use crate::metrics::{ExecutionPlanMetricsSet, SpillMetrics};use crate::spill::{get_record_batch_memory_size, spill_manager::GetSlicedSize};+ use crate::stream::RecordBatchStreamAdapter;use arrow::datatypes::{DataType, Field, Schema};use arrow::{array::{ArrayRef, Int32Array, StringArray, StringViewArray},record_batch::RecordBatch,};- use datafusion_common::Result;- use datafusion_execution::runtime_env::RuntimeEnv;+ use async_trait::async_trait;+ use bytes::Bytes;+ use datafusion_common::{DataFusionError, Result, not_impl_err};+ use datafusion_execution::disk_manager::DiskManagerBuilder;+ use datafusion_execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};+ use datafusion_execution::{+ AsyncSpillWriter, SendableRecordBatchStream, SpillFile, SpillWriter,+ TempFileFactory,+ };+ use futures::Stream;+ use std::path::Path;+ use std::pin::Pin;use std::sync::Arc;+ use std::sync::atomic::{AtomicUsize, Ordering};++ struct AbortTrackingTempFileFactory {+ abort_count: Arc<AtomicUsize>,+ }++ impl TempFileFactory for AbortTrackingTempFileFactory {+ fn create_temp_file(&self, _description: &str) -> Result<Arc<dyn SpillFile>> {+ Ok(Arc::new(AbortTrackingSpillFile {+ abort_count: Arc::clone(&self.abort_count),+ }))+ }+ }++ struct AbortTrackingSpillFile {+ abort_count: Arc<AtomicUsize>,+ }++ #[async_trait]+ impl SpillFile for AbortTrackingSpillFile {+ fn path(&self) -> Option<&Path> {+ None+ }++ fn size(&self) -> Option<u64> {+ Some(0)+ }++ fn read_stream(+ &self,+ ) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes>> + Send>>> {+ Ok(Box::pin(futures::stream::empty()))+ }++ fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {+ not_impl_err!("test backend only supports asynchronous writes")+ }++ async fn open_async_writer(&self) -> Result<Box<dyn AsyncSpillWriter>> {+ Ok(Box::new(AbortTrackingWriter {+ abort_count: Arc::clone(&self.abort_count),+ }))+ }+ }++ struct AbortTrackingWriter {+ abort_count: Arc<AtomicUsize>,+ }++ #[async_trait]+ impl AsyncSpillWriter for AbortTrackingWriter {+ async fn write_all(&mut self, _data: Bytes) -> Result<()> {+ Ok(())+ }++ async fn finish(&mut self) -> Result<()> {+ Ok(())+ }++ async fn abort(&mut self) -> Result<()> {+ self.abort_count.fetch_add(1, Ordering::Relaxed);+ Ok(())+ }+ }fn build_test_spill_manager(env: Arc<RuntimeEnv>,@@ -357,6 +444,46 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn test_stream_spill_aborts_after_input_error() -> Result<()> {+ let schema = Arc::new(Schema::new(vec![+ Field::new("id", DataType::Int32, false),+ Field::new("value", DataType::Utf8, false),+ ]));+ let abort_count = Arc::new(AtomicUsize::new(0));+ let disk_manager_builder = DiskManagerBuilder::default().with_temp_file_factory(+ Arc::new(AbortTrackingTempFileFactory {+ abort_count: Arc::clone(&abort_count),+ }),+ );+ let env = RuntimeEnvBuilder::new()+ .with_disk_manager_builder(disk_manager_builder)+ .build_arc()?;+ let manager = build_test_spill_manager(env, Arc::clone(&schema));+ let batch = build_writer_batch(Arc::clone(&schema))?;+ let input = futures::stream::iter(vec![+ Ok(batch),+ Err(DataFusionError::Execution(+ "injected input stream failure".to_string(),+ )),+ ]);+ let mut stream: SendableRecordBatchStream =+ Box::pin(RecordBatchStreamAdapter::new(schema, input));++ let error = manager+ .spill_record_batch_stream_and_return_max_batch_memory(+ &mut stream,+ "abort-test",+ )+ .await+ .err()+ .expect("the injected stream error should be returned");++ assert!(error.to_string().contains("injected input stream failure"));+ assert_eq!(abort_count.load(Ordering::Relaxed), 1);+ Ok(())+ }+#[test]fn check_sliced_size_for_string_view_array() -> Result<()> {let array_length = 50;