Apache DataFusion SQL Query Engine
feat: introduce pluggable SpillFile trait and TempFileFactory for custom spill backends (#21882)
pantShrey01bf68cdc46f
diff --git a/Cargo.lock b/Cargo.lockindex 098a0afb98..c3ba2cc236 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2106,6 +2106,7 @@ dependencies = ["arrow","arrow-buffer","async-trait",+ "bytes","chrono","dashmap","datafusion-common",@@ -2119,6 +2120,8 @@ dependencies = ["parquet","rand 0.9.4","tempfile",+ "tokio",+ "tokio-util","url",]@@ -2448,6 +2451,7 @@ dependencies = ["arrow-ord","arrow-schema","async-trait",+ "bytes","criterion","datafusion-common","datafusion-common-runtime",diff --git a/datafusion/execution/Cargo.toml b/datafusion/execution/Cargo.tomlindex 06c84d8acb..0aa2739e35 100644--- a/datafusion/execution/Cargo.toml+++ b/datafusion/execution/Cargo.toml@@ -55,6 +55,7 @@ sql = []arrow = { workspace = true }arrow-buffer = { workspace = true }async-trait = { workspace = true }+bytes = { workspace = true }dashmap = { workspace = true }datafusion-common = { workspace = true, default-features = false }datafusion-expr = { workspace = true, default-features = false }@@ -66,7 +67,11 @@ parking_lot = { workspace = true }parquet = { workspace = true, optional = true }rand = { workspace = true }tempfile = { workspace = true }+tokio = { workspace = true }+tokio-util = { workspace = true, features = ["io"] }url = { workspace = true }+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]+tokio = { workspace = true, features = ["fs"] }[dev-dependencies]chrono = { workspace = true }diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rsindex c3c8559e4b..ff8403d916 100644--- a/datafusion/execution/src/disk_manager.rs+++ b/datafusion/execution/src/disk_manager.rs@@ -17,23 +17,24 @@//! [`DiskManager`]: Manages files generated during query execution-use datafusion_common::{- DataFusionError, Result, config_err, resources_datafusion_err, resources_err,-};+use crate::spill_file::{SpillFile, SpillWriter, TempFileFactory};+use bytes::Bytes;+use datafusion_common::human_readable_size;+use datafusion_common::{DataFusionError, Result, config_err, resources_datafusion_err};+#[cfg(not(target_arch = "wasm32"))]+use futures::StreamExt;use log::debug;use parking_lot::Mutex;use rand::{Rng, rng};+use std::fmt::Debug;use std::path::{Path, PathBuf};use std::sync::Arc;use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};use tempfile::{Builder, NamedTempFile, TempDir};--use datafusion_common::human_readable_size;-pub const DEFAULT_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB/// Builder pattern for the [DiskManager] structure-#[derive(Clone, Debug)]+#[derive(Clone)]pub struct DiskManagerBuilder {/// The storage mode of the disk managermode: DiskManagerMode,@@ -41,7 +42,14 @@ pub struct DiskManagerBuilder {/// Default to 100GBmax_temp_directory_size: u64,}-+impl Debug for DiskManagerBuilder {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("DiskManagerBuilder")+ .field("mode", &self.mode)+ .field("max_temp_directory_size", &self.max_temp_directory_size)+ .finish()+ }+}impl Default for DiskManagerBuilder {fn default() -> Self {Self {@@ -78,6 +86,7 @@ impl DiskManagerBuilder {max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),+ factory: None,}),DiskManagerMode::Directories(conf_dirs) => {let local_dirs = create_local_dirs(&conf_dirs)?;@@ -89,6 +98,7 @@ impl DiskManagerBuilder {max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),+ factory: None,})}DiskManagerMode::Disabled => Ok(DiskManager {@@ -96,12 +106,20 @@ impl DiskManagerBuilder {max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),+ factory: None,+ }),+ DiskManagerMode::Custom(factory) => Ok(DiskManager {+ local_dirs: Mutex::new(None),+ max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ used_disk_space: Arc::new(AtomicU64::new(0)),+ active_files_count: Arc::new(AtomicUsize::new(0)),+ factory: Some(factory),}),}}}-#[derive(Clone, Debug, Default)]+#[derive(Clone, Default)]pub enum DiskManagerMode {/// Create a new [DiskManager] that creates temporary files within/// a temporary directory chosen by the OS@@ -113,13 +131,26 @@ pub enum DiskManagerMode {/// at random for each temporary file created.Directories(Vec<PathBuf>),+ /// Create a new [DiskManager] with a cutstom backend+ Custom(Arc<dyn TempFileFactory>),+/// Disable disk manager, attempts to create temporary files will errorDisabled,}+impl Debug for DiskManagerMode {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::OsTmpDirectory => write!(f, "OsTmpDirectory"),+ Self::Directories(dirs) => f.debug_tuple("Directories").field(dirs).finish(),+ Self::Disabled => write!(f, "Disabled"),+ Self::Custom(_) => write!(f, "Custom(Arc<dyn TempFileFactory>)"),+ }+ }+}+/// Manages files generated during query execution, e.g. spill files generated/// while processing dataset larger than available memory.-#[derive(Debug)]pub struct DiskManager {/// TempDirs to put temporary files in.///@@ -135,8 +166,20 @@ pub struct DiskManager {used_disk_space: Arc<AtomicU64>,/// Number of active temporary files created by this disk manageractive_files_count: Arc<AtomicUsize>,+ /// Factory+ factory: Option<Arc<dyn TempFileFactory>>,+}+impl Debug for DiskManager {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("DiskManager")+ .field("local_dirs", &self.local_dirs)+ .field("max_temp_directory_size", &self.max_temp_directory_size)+ .field("used_disk_space", &self.used_disk_space)+ .field("active_files_count", &self.active_files_count)+ .field("factory", &self.factory.is_some())+ .finish()+ }}-/// Information about the current disk usage for spilling#[derive(Debug, Clone, Copy)]pub struct SpillingProgress {@@ -163,7 +206,12 @@ impl DiskManager {&self,max_temp_directory_size: u64,) -> Result<()> {- if self.local_dirs.lock().is_none() && max_temp_directory_size != 0 {+ // If the disk manager is disabled and `max_temp_directory_size` is not 0,+ // this operation is not meaningful, fail early.+ if self.local_dirs.lock().is_none()+ && max_temp_directory_size != 0+ && self.factory.is_none()+ {return config_err!("Cannot set max temp directory size for a disk manager that spilling is disabled");@@ -227,7 +275,7 @@ impl DiskManager {/// files. If this returns false, any call to `create_tmp_file`/// will error.pub fn tmp_files_enabled(&self) -> bool {- self.local_dirs.lock().is_some()+ self.factory.is_some() || self.local_dirs.lock().is_some()}/// Return a temporary file from a randomized choice in the configured locations@@ -237,7 +285,11 @@ impl DiskManager {pub fn create_tmp_file(self: &Arc<Self>,request_description: &str,- ) -> Result<RefCountedTempFile> {+ ) -> Result<Arc<dyn SpillFile>> {+ // Delegate to custom backend if configured+ if let Some(factory) = &self.factory {+ return factory.create_temp_file(request_description);+ }let mut guard = self.local_dirs.lock();let local_dirs = guard.as_mut().ok_or_else(|| {resources_datafusion_err!(@@ -260,7 +312,7 @@ impl DiskManager {let dir_index = rng().random_range(0..local_dirs.len());self.active_files_count.fetch_add(1, Ordering::Relaxed);- Ok(RefCountedTempFile {+ Ok(Arc::new(RefCountedTempFile {parent_temp_dir: Arc::clone(&local_dirs[dir_index]),tempfile: Arc::new(Builder::new()@@ -269,19 +321,13 @@ impl DiskManager {),current_file_disk_usage: Arc::new(AtomicU64::new(0)),disk_manager: Arc::clone(self),- })+ }))}}/// A wrapper around a [`NamedTempFile`] that also contains/// a reference to its parent temporary directory.///-/// # Note-/// After any modification to the underlying file (e.g., writing data to it), the caller-/// must invoke [`Self::update_disk_usage`] to update the global disk usage counter.-/// This ensures the disk manager can properly enforce usage limits configured by-/// [`DiskManager::with_max_temp_directory_size`].-////// This type is Clone-able, allowing multiple references to the same underlying file./// The file is deleted only when the last reference is dropped.///@@ -297,8 +343,7 @@ pub struct RefCountedTempFile {parent_temp_dir: Arc<TempDir>,/// The underlying temporary file, wrapped in Arc to allow cloningtempfile: Arc<NamedTempFile>,- /// Tracks the current disk usage of this temporary file. See- /// [`Self::update_disk_usage`] for more details.+ /// Tracks the current disk usage of this temporary file.////// This is wrapped in `Arc<AtomicU64>` so that all clones share the same/// disk usage tracking, preventing incorrect accounting when clones are dropped.@@ -327,59 +372,7 @@ impl RefCountedTempFile {self.tempfile.as_ref()}- /// Updates the global disk usage counter after modifications to the underlying file.- ///- /// # Errors- /// - Returns an error if the global disk usage exceeds the configured limit.- pub fn update_disk_usage(&mut self) -> Result<()> {- // Get new file size from OS- let metadata = self.tempfile.as_file().metadata()?;- let new_disk_usage = metadata.len();-- // Get the old disk usage- let old_disk_usage = self.current_file_disk_usage.load(Ordering::Relaxed);-- // Update the global disk usage by:- // 1. Subtracting the old file size from the global counter- self.disk_manager- .used_disk_space- .fetch_sub(old_disk_usage, Ordering::Relaxed);- // 2. Adding the new file size to the global counter- self.disk_manager- .used_disk_space- .fetch_add(new_disk_usage, Ordering::Relaxed);-- // 3. Check if the updated global disk usage exceeds the configured limit- let global_disk_usage = self.disk_manager.used_disk_space.load(Ordering::Relaxed);- let limit = self- .disk_manager- .max_temp_directory_size- .load(Ordering::Relaxed);- if global_disk_usage > limit {- // Roll back: restore global counter to previous state so that- // Drop (which subtracts current_file_disk_usage = old value) remains- // consistent. Without this, the delta (new - old) leaks permanently.- self.disk_manager- .used_disk_space- .fetch_sub(new_disk_usage, Ordering::Relaxed);- self.disk_manager- .used_disk_space- .fetch_add(old_disk_usage, Ordering::Relaxed);- return resources_err!(- "The used disk space during the spilling process has exceeded the allowable limit of {}. \- Please try increasing the config: `datafusion.runtime.max_temp_directory_size`.",- human_readable_size(limit as usize)- );- }-- // 4. Update the local file size tracking- self.current_file_disk_usage- .store(new_disk_usage, Ordering::Relaxed);-- Ok(())- }-- pub fn current_disk_usage(&self) -> u64 {+ fn current_disk_usage(&self) -> u64 {self.current_file_disk_usage.load(Ordering::Relaxed)}}@@ -419,6 +412,122 @@ fn create_local_dirs(local_dirs: &[PathBuf]) -> Result<Vec<Arc<TempDir>>> {.collect()}+pub struct FileSpillWriter {+ file: std::fs::File,+ disk_manager: Arc<DiskManager>,+ current_file_disk_usage: Arc<AtomicU64>,+}++impl std::io::Write for FileSpillWriter {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ let len = buf.len() as u64;+ if len == 0 {+ return Ok(0);+ }++ let new_global = self+ .disk_manager+ .used_disk_space+ .fetch_add(len, Ordering::Relaxed)+ + len;++ let limit = self.disk_manager.max_temp_directory_size();++ if new_global > limit {+ self.disk_manager+ .used_disk_space+ .fetch_sub(len, Ordering::Relaxed);++ return Err(std::io::Error::other(format!(+ "The used disk space during the spilling process has exceeded the allowable limit of {}. \+ Please try increasing the config: `datafusion.runtime.max_temp_directory_size`.",+ human_readable_size(limit as usize)+ )));+ }++ self.file.write_all(buf).map_err(DataFusionError::IoError)?;++ self.current_file_disk_usage+ .fetch_add(len, Ordering::Relaxed);++ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ self.file.flush()+ }+}++impl SpillWriter for FileSpillWriter {+ fn finish(&mut self) -> Result<()> {+ // flush() is handled by Arrow, nothing left to do here+ Ok(())+ }+}++impl SpillFile for RefCountedTempFile {+ fn path(&self) -> Option<&Path> {+ Some(self.tempfile.path())+ }++ fn size(&self) -> Option<u64> {+ Some(self.current_disk_usage())+ }+ #[cfg(not(target_arch = "wasm32"))]+ fn read_stream(+ &self,+ ) -> Result<std::pin::Pin<Box<dyn futures::Stream<Item = Result<Bytes>> + Send>>>+ {+ let path = self.path().to_owned();++ let stream =+ futures::stream::once(async move {+ tokio::fs::File::open(&path)+ .await+ .map_err(DataFusionError::IoError)+ })+ .flat_map(+ |open_result| -> std::pin::Pin<+ Box<dyn futures::Stream<Item = Result<Bytes>> + Send>,+ > {+ match open_result {+ Ok(file) => Box::pin(+ // Use a 128KB read buffer. The default 8KB causes excessive async+ // poll overhead when reading multi-MB spill files back into memory.+ tokio_util::io::ReaderStream::with_capacity(file, 128 * 1024)+ .map(|r| r.map_err(DataFusionError::IoError)),+ ),+ Err(e) => Box::pin(futures::stream::once(async move { Err(e) })),+ }+ },+ );++ Ok(Box::pin(stream))+ }++ #[cfg(target_arch = "wasm32")]+ fn read_stream(+ &self,+ ) -> Result<std::pin::Pin<Box<dyn futures::Stream<Item = Result<Bytes>> + Send>>>+ {+ datafusion_common::exec_err!(+ "Default OS file spilling is not supported on WASM. Configure DiskManager with a Custom TempFileFactory."+ )+ }++ fn open_writer(&self) -> Result<Box<dyn SpillWriter>> {+ let file = self+ .tempfile+ .as_file()+ .try_clone()+ .map_err(DataFusionError::IoError)?;+ Ok(Box::new(FileSpillWriter {+ file,+ disk_manager: Arc::clone(&self.disk_manager),+ current_file_disk_usage: Arc::clone(&self.current_file_disk_usage),+ }))+ }+}#[cfg(test)]mod tests {use super::*;@@ -438,7 +547,10 @@ mod tests {// the returned tempfile file should be in the temp directorylet local_dirs = local_dir_snapshot(&dm);- assert_path_in_dirs(actual.path(), local_dirs.iter().map(|p| p.as_path()));+ assert_path_in_dirs(+ actual.path().unwrap(),+ local_dirs.iter().map(|p| p.as_path()),+ );Ok(())}@@ -470,7 +582,7 @@ mod tests {let actual = dm.create_tmp_file("Testing")?;// the file should be in one of the specified local directories- assert_path_in_dirs(actual.path(), local_dirs.into_iter());+ assert_path_in_dirs(actual.path().unwrap(), local_dirs.into_iter());Ok(())}@@ -484,13 +596,17 @@ mod tests {.unwrap(),);assert!(!manager.tmp_files_enabled());- assert_eq!(- manager- .create_tmp_file("Testing")- .unwrap_err()- .strip_backtrace(),- "Resources exhausted: Memory Exhausted while Testing (DiskManager is disabled)",- )+ match manager.create_tmp_file("Testing") {+ Err(e) => {+ assert_eq!(+ e.strip_backtrace(),+ "Resources exhausted: Memory Exhausted while Testing (DiskManager is disabled)"+ );+ }+ Ok(_) => {+ panic!("Expected DiskManager to fail creating a file when disabled!")+ }+ }}#[test]@@ -523,7 +639,7 @@ mod tests {// Test for the case using OS arranged temporary directorylet dm = Arc::new(DiskManagerBuilder::default().build()?);let temp_file = dm.create_tmp_file("Testing")?;- let temp_file_path = temp_file.path().to_owned();+ let temp_file_path = temp_file.path().unwrap().to_owned();assert!(temp_file_path.exists());drop(dm);@@ -545,7 +661,7 @@ mod tests {.build()?,);let temp_file = dm.create_tmp_file("Testing")?;- let temp_file_path = temp_file.path().to_owned();+ let temp_file_path = temp_file.path().unwrap().to_owned();assert!(temp_file_path.exists());drop(dm);@@ -559,30 +675,26 @@ mod tests {#[test]fn test_disk_usage_basic() -> Result<()> {- use std::io::Write;-let dm = Arc::new(DiskManagerBuilder::default().build()?);- let mut temp_file = dm.create_tmp_file("Testing")?;-+ let temp_file = dm.create_tmp_file("Testing")?;+ let mut writer = temp_file.open_writer()?;// Initially, disk usage should be 0assert_eq!(dm.used_disk_space(), 0);- assert_eq!(temp_file.current_disk_usage(), 0);+ assert_eq!(temp_file.size().unwrap(), 0);// Write some data to the file- temp_file.inner().as_file().write_all(b"hello world")?;- temp_file.update_disk_usage()?;+ writer.write_all(b"hello world")?;// Disk usage should now reflect the written data- let expected_usage = temp_file.current_disk_usage();+ let expected_usage = temp_file.size().unwrap();assert!(expected_usage > 0);assert_eq!(dm.used_disk_space(), expected_usage);// Write more data- temp_file.inner().as_file().write_all(b" more data")?;- temp_file.update_disk_usage()?;+ writer.write_all(b"more_data")?;// Disk usage should increase- let new_usage = temp_file.current_disk_usage();+ let new_usage = temp_file.size().unwrap();assert!(new_usage > expected_usage);assert_eq!(dm.used_disk_space(), new_usage);@@ -597,64 +709,60 @@ mod tests {#[test]fn test_disk_usage_with_clones() -> Result<()> {- use std::io::Write;-let dm = Arc::new(DiskManagerBuilder::default().build()?);- let mut temp_file = dm.create_tmp_file("Testing")?;+ let temp_file = dm.create_tmp_file("Testing")?;// Write some data- temp_file.inner().as_file().write_all(b"test data")?;- temp_file.update_disk_usage()?;+ let mut writer = temp_file.open_writer()?;+ writer.write_all(b"test data")?;- let usage_after_write = temp_file.current_disk_usage();+ let usage_after_write = temp_file.size().unwrap();assert!(usage_after_write > 0);assert_eq!(dm.used_disk_space(), usage_after_write);// Clone the file- let clone1 = temp_file.clone();- let clone2 = temp_file.clone();+ let clone1 = Arc::clone(&temp_file);+ let clone2 = Arc::clone(&temp_file);// All clones should see the same disk usage- assert_eq!(clone1.current_disk_usage(), usage_after_write);- assert_eq!(clone2.current_disk_usage(), usage_after_write);-+ assert_eq!(clone1.size().unwrap(), usage_after_write);+ assert_eq!(clone2.size().unwrap(), usage_after_write);// Global disk usage should still be the same (not multiplied by number of clones)assert_eq!(dm.used_disk_space(), usage_after_write);// Write more data through one clone- clone1.inner().as_file().write_all(b" more data")?;- let mut mutable_clone1 = clone1;- mutable_clone1.update_disk_usage()?;+ let mut clone_writer = clone1.open_writer()?;+ clone_writer.write_all(b" more data")?;- let new_usage = mutable_clone1.current_disk_usage();+ let new_usage = clone1.size().unwrap();assert!(new_usage > usage_after_write);-// All clones should see the updated disk usage- assert_eq!(temp_file.current_disk_usage(), new_usage);- assert_eq!(clone2.current_disk_usage(), new_usage);- assert_eq!(mutable_clone1.current_disk_usage(), new_usage);+ assert_eq!(temp_file.size().unwrap(), new_usage);+ assert_eq!(clone2.size().unwrap(), new_usage);+ assert_eq!(clone1.size().unwrap(), new_usage);// Global disk usage should reflect the new size (not multiplied)assert_eq!(dm.used_disk_space(), new_usage);// Drop one clone- drop(mutable_clone1);+ drop(clone_writer);+ drop(clone1);// Disk usage should NOT change (other clones still exist)assert_eq!(dm.used_disk_space(), new_usage);- assert_eq!(temp_file.current_disk_usage(), new_usage);- assert_eq!(clone2.current_disk_usage(), new_usage);+ assert_eq!(temp_file.size().unwrap(), new_usage);+ assert_eq!(clone2.size().unwrap(), new_usage);// Drop another clonedrop(clone2);// Disk usage should still NOT change (original still exists)assert_eq!(dm.used_disk_space(), new_usage);- assert_eq!(temp_file.current_disk_usage(), new_usage);+ assert_eq!(temp_file.size().unwrap(), new_usage);// Drop the original+ drop(writer);drop(temp_file);-// Now disk usage should return to 0 (last reference dropped)assert_eq!(dm.used_disk_space(), 0);@@ -663,29 +771,27 @@ mod tests {#[test]fn test_disk_usage_clones_dropped_out_of_order() -> Result<()> {- use std::io::Write;-let dm = Arc::new(DiskManagerBuilder::default().build()?);- let mut temp_file = dm.create_tmp_file("Testing")?;+ let temp_file = dm.create_tmp_file("Testing")?;+ let mut writer = temp_file.open_writer()?;// Write data- temp_file.inner().as_file().write_all(b"test")?;- temp_file.update_disk_usage()?;+ writer.write_all(b"test")?;- let usage = temp_file.current_disk_usage();+ let usage = temp_file.size().unwrap();assert_eq!(dm.used_disk_space(), usage);// Create multiple clones- let clone1 = temp_file.clone();- let clone2 = temp_file.clone();- let clone3 = temp_file.clone();+ let clone1 = Arc::clone(&temp_file);+ let clone2 = Arc::clone(&temp_file);+ let clone3 = Arc::clone(&temp_file);// Drop the original first (out of order)drop(temp_file);// Disk usage should still be tracked (clones exist)assert_eq!(dm.used_disk_space(), usage);- assert_eq!(clone1.current_disk_usage(), usage);+ assert_eq!(clone1.size().unwrap(), usage);// Drop clones in different orderdrop(clone2);@@ -705,25 +811,24 @@ mod tests {#[test]fn test_disk_usage_multiple_files() -> Result<()> {- use std::io::Write;-let dm = Arc::new(DiskManagerBuilder::default().build()?);// Create multiple temp files- let mut file1 = dm.create_tmp_file("Testing1")?;- let mut file2 = dm.create_tmp_file("Testing2")?;+ let file1 = dm.create_tmp_file("Testing1")?;+ let file2 = dm.create_tmp_file("Testing2")?;++ let mut writer1 = file1.open_writer()?;+ let mut writer2 = file2.open_writer()?;// Write to first file- file1.inner().as_file().write_all(b"file1")?;- file1.update_disk_usage()?;- let usage1 = file1.current_disk_usage();+ writer1.write_all(b"file1")?;+ let usage1 = file1.size().unwrap();assert_eq!(dm.used_disk_space(), usage1);// Write to second file- file2.inner().as_file().write_all(b"file2 data")?;- file2.update_disk_usage()?;- let usage2 = file2.current_disk_usage();+ writer2.write_all(b"file2 data")?;+ let usage2 = file2.size().unwrap();// Global usage should be sum of both filesassert_eq!(dm.used_disk_space(), usage1 + usage2);@@ -933,8 +1038,6 @@ mod tests {//// Without the rollback fix, the global counter would be permanently// inflated by the delta between the new and old file sizes.- use std::fs::OpenOptions;- use std::io::Write;let dm = Arc::new(DiskManager::builder()@@ -942,30 +1045,25 @@ mod tests {.build()?,);- // Create a temp file and write some data via a separate writable handle- let mut file = dm.create_tmp_file("test_rollback")?;+ let file = dm.create_tmp_file("test_rollback")?;++ let mut writer = file.open_writer()?;++ // Create a temp file and write some data{- let path = file.path().to_path_buf();- let mut f = OpenOptions::new().append(true).open(&path)?;let data = vec![0u8; 1024]; // 1KB- f.write_all(&data)?;- f.sync_all()?;+ writer.write_all(&data)?;}- // Record the file's disk usage- file.update_disk_usage()?;+let usage_after_first_write = dm.used_disk_space();assert!(usage_after_first_write > 0);// Write more data to grow the file{- let path = file.path().to_path_buf();- let mut f = OpenOptions::new().append(true).open(&path)?;let data = vec![0u8; 4 * 1024]; // 4KB more- f.write_all(&data)?;- f.sync_all()?;+ writer.write_all(&data)?;}- // Update disk usage — should succeed (still under 10MB)- file.update_disk_usage()?;+let usage_after_second_write = dm.used_disk_space();assert!(usage_after_second_write > usage_after_first_write);@@ -974,28 +1072,26 @@ mod tests {// Write even more data{- let path = file.path().to_path_buf();- let mut f = OpenOptions::new().append(true).open(&path)?;let data = vec![0u8; 2 * 1024]; // 2KB more- f.write_all(&data)?;- f.sync_all()?;- }- // This update_disk_usage should FAIL (exceeds new 1-byte limit)- let result = file.update_disk_usage();- assert!(result.is_err());- assert!(- result- .unwrap_err()- .to_string()- .contains("exceeded the allowable limit")- );+ // This write should FAIL (exceeds new 1-byte limit)+ let result = writer.write_all(&data);++ assert!(result.is_err());+ assert!(+ result+ .unwrap_err()+ .to_string()+ .contains("exceeded the allowable limit")+ );+ }// Critical check: used_disk_space should still equal the LAST// successful update (before the failed one), not be inflatedassert_eq!(dm.used_disk_space(), usage_after_second_write);- // Drop the file — should subtract the last successful file size+ // Drop the writer and file — should subtract the last successful file size+ drop(writer);drop(file);// After drop: used_disk_space must be zero (no leak)diff --git a/datafusion/execution/src/lib.rs b/datafusion/execution/src/lib.rsindex 1a8da9459a..5c646066ed 100644--- a/datafusion/execution/src/lib.rs+++ b/datafusion/execution/src/lib.rs@@ -35,9 +35,9 @@ pub mod object_store;#[cfg(feature = "parquet_encryption")]pub mod parquet_encryption;pub mod runtime_env;+pub mod spill_file;mod stream;mod task;-pub mod registry {pub use datafusion_expr::registry::{FunctionRegistry, MemoryFunctionRegistry, SerializerRegistry,@@ -46,5 +46,6 @@ pub mod registry {pub use disk_manager::DiskManager;pub use registry::FunctionRegistry;+pub use spill_file::{SpillFile, SpillWriter, TempFileFactory};pub use stream::{RecordBatchStream, SendableRecordBatchStream};pub use task::{TaskContext, TaskContextProvider};diff --git a/datafusion/execution/src/spill_file.rs b/datafusion/execution/src/spill_file.rsnew file mode 100644index 0000000000..dea54dd5d2--- /dev/null+++ b/datafusion/execution/src/spill_file.rs@@ -0,0 +1,52 @@+// Licensed to the Apache Software Foundation (ASF) under one+// or more contributor license agreements. See the NOTICE file+// distributed with this work for additional information+// regarding copyright ownership. The ASF licenses this file+// to you under the Apache License, Version 2.0 (the+// "License"); you may not use this file except in compliance+// with the License. You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing,+// software distributed under the License is distributed on an+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+// KIND, either express or implied. See the License for the+// specific language governing permissions and limitations+// under the License.++use bytes::Bytes;+use datafusion_common::Result;+use futures::Stream;+use std::path::Path;+use std::pin::Pin;+use std::sync::Arc;++/// Abstraction over a spill file backend.+/// Implementations handle their own quota enforcement and blocking concerns.+pub trait SpillFile: Send + Sync {+ /// Returns the OS path if this is a local file, None otherwise.+ fn path(&self) -> Option<&Path> {+ None+ }++ /// Returns current size in bytes if cheaply available.+ fn size(&self) -> Option<u64>;++ /// Returns file contents as an async stream of byte chunks.+ fn read_stream(&self) -> Result<Pin<Box<dyn Stream<Item = Result<Bytes>> + Send>>>;++ /// Opens a writer for appending data to this file.+ fn open_writer(&self) -> Result<Box<dyn SpillWriter>>;+}++/// Writer for spill file backends.+pub trait SpillWriter: std::io::Write + Send {+ /// Intended for close/sync/commit operations.+ fn finish(&mut self) -> Result<()>;+}++/// Factory for creating spill files.+pub trait TempFileFactory: Send + Sync {+ fn create_temp_file(&self, description: &str) -> Result<Arc<dyn SpillFile>>;+}diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.tomlindex 0e0b7e3b24..c43ae81003 100644--- a/datafusion/physical-plan/Cargo.toml+++ b/datafusion/physical-plan/Cargo.toml@@ -65,6 +65,7 @@ arrow-ipc = { workspace = true, features = ["lz4", "zstd"] }arrow-ord = { workspace = true }arrow-schema = { workspace = true }async-trait = { workspace = true }+bytes = { workspace = true }datafusion-common = { workspace = true }datafusion-common-runtime = { workspace = true, default-features = true }datafusion-execution = { workspace = true }diff --git a/datafusion/physical-plan/benches/spill_io.rs b/datafusion/physical-plan/benches/spill_io.rsindex fac2547a13..ddd83ca565 100644--- a/datafusion/physical-plan/benches/spill_io.rs+++ b/datafusion/physical-plan/benches/spill_io.rs@@ -547,7 +547,7 @@ fn benchmark_spill_batches_for_all_codec(let write_throughput = (mem_bytes as u128 / write_time.as_millis().max(1)) * 1000;// calculate compression ratio- let disk_bytes = std::fs::metadata(spill_file.path())+ let disk_bytes = std::fs::metadata(spill_file.path().unwrap()).expect("metadata read fail").len() as usize;let ratio = mem_bytes as f64 / disk_bytes.max(1) as f64;diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rsindex db552fed96..3cae05a3a8 100644--- a/datafusion/physical-plan/src/joins/nested_loop_join.rs+++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs@@ -66,9 +66,8 @@ use datafusion_common::{assert_eq_or_internal_err, internal_datafusion_err, internal_err, project_schema,unwrap_or_internal_err,};-use datafusion_execution::TaskContext;-use datafusion_execution::disk_manager::RefCountedTempFile;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation};+use datafusion_execution::{SpillFile, TaskContext};use datafusion_expr::JoinType;use datafusion_physical_expr::equivalence::{ProjectionMapping, join_equivalence_properties,@@ -903,7 +902,7 @@ pub(crate) struct LeftSpillData {/// SpillManager used to read the spill file (has the left schema)spill_manager: SpillManager,/// The spill file containing all left-side batches- spill_file: RefCountedTempFile,+ spill_file: Arc<dyn SpillFile>,/// Left-side schemaschema: SchemaRef,}@@ -1581,7 +1580,7 @@ impl NestedLoopJoinStream {Poll::Ready(Ok(spill_data)) => {match spill_data.spill_manager- .read_spill_as_stream(spill_data.spill_file.clone(), None)+ .read_spill_as_stream(Arc::clone(&spill_data.spill_file), None){Ok(stream) => {active.left_schema = Some(Arc::clone(&spill_data.schema));diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rsindex 8de72fd49c..d1ca9707fe 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/bitwise_stream.rs@@ -137,9 +137,8 @@ use arrow::util::bit_util::apply_bitwise_binary_op;use datafusion_common::{JoinSide, JoinType, NullEquality, Result, ScalarValue, internal_err,};-use datafusion_execution::SendableRecordBatchStream;-use datafusion_execution::disk_manager::RefCountedTempFile;use datafusion_execution::memory_pool::MemoryReservation;+use datafusion_execution::{SendableRecordBatchStream, SpillFile};use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;use futures::{Stream, StreamExt, ready};@@ -254,7 +253,7 @@ pub(crate) struct BitwiseSortMergeJoinStream {// with many inner rows will buffer them all. See "Degenerate cases"// in exec.rs. Spilled to disk when memory reservation fails.inner_key_buffer: Vec<RecordBatch>,- inner_key_spill: Option<RefCountedTempFile>,+ inner_key_spill: Option<Arc<dyn SpillFile>>,// Track the active spill_streamspill_stream: Option<SendableRecordBatchStream>,@@ -808,7 +807,7 @@ impl BitwiseSortMergeJoinStream {{let stream = self.spill_manager- .read_spill_as_stream(spill_file.clone(), None)?;+ .read_spill_as_stream(Arc::clone(spill_file), None)?;self.spill_stream = Some(stream);}diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rsindex f1a18aac76..51cf38b9ab 100644--- a/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs+++ b/datafusion/physical-plan/src/joins/sort_merge_join/materializing_stream.rs@@ -51,7 +51,7 @@ use arrow::compute::{use arrow::datatypes::SchemaRef;use datafusion_common::cast::as_uint64_array;use datafusion_common::{JoinType, NullEquality, Result, exec_err, internal_err};-use datafusion_execution::disk_manager::RefCountedTempFile;+use datafusion_execution::SpillFile;use datafusion_execution::memory_pool::MemoryReservation;use datafusion_execution::runtime_env::RuntimeEnv;use datafusion_physical_expr_common::physical_expr::PhysicalExprRef;@@ -221,7 +221,7 @@ pub(super) enum FilterState {/// A buffered batch that contains contiguous rows with same join key///-/// `BufferedBatch` can exist as either an in-memory `RecordBatch` or a `RefCountedTempFile` on disk.+/// `BufferedBatch` can exist as either an in-memory `RecordBatch` or a `SpillFile`.#[derive(Debug)]pub(super) struct BufferedBatch {/// Represents in memory or spilled record batch@@ -297,14 +297,23 @@ impl BufferedBatch {// TODO: Spill join arrays (https://github.com/apache/datafusion/pull/17429)// Used to represent whether the buffered data is currently in memory or written to disk-#[derive(Debug)]pub(super) enum BufferedBatchState {// In memory record batchInMemory(RecordBatch),// Spilled temp file- Spilled(RefCountedTempFile),+ Spilled(Arc<dyn SpillFile>),}+impl Debug for BufferedBatchState {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::InMemory(batch) => f.debug_tuple("InMemory").field(batch).finish(),+ Self::Spilled(_) => {+ write!(f, "Spilled(Custom_Backend)")+ }+ }+ }+}/// Sort-Merge join stream for Inner/Left/Right/Full joins.////// Named "materializing" because it builds explicit `(streamed, buffered)` row@@ -989,7 +998,7 @@ impl MaterializingSortMergeJoinStream {if self.spill_stream.is_none() {let stream = self.spill_manager- .read_spill_as_stream(spill_file.clone(), None)?;+ .read_spill_as_stream(Arc::clone(spill_file), None)?;self.spill_stream = Some(stream);}diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rsindex 2298183485..1059f0225f 100644--- a/datafusion/physical-plan/src/repartition/mod.rs+++ b/datafusion/physical-plan/src/repartition/mod.rs@@ -3100,9 +3100,9 @@ mod test {let input_partitions = vec![partition1, partition2];// Set up context with tight memory limit to force spilling- // Sorting needs some non-spillable memory, so 64 bytes should force spilling while still allowing the query to complete+ // Sorting needs some non-spillable memory, so 608 bytes should force spilling while still allowing the query to completelet runtime = RuntimeEnvBuilder::default()- .with_memory_limit(64, 1.0)+ .with_memory_limit(608, 1.0).build_arc()?;let task_ctx = TaskContext::default().with_runtime(runtime);@@ -3167,38 +3167,13 @@ mod test {assert_batches_eq!(expected, std::slice::from_ref(batch));}- // We should have spilled ~ all of the data.- // - We spill data during the repartitioning phase- // - We may also spill during the final merge sort- let all_batches = [batch1, batch2, batch3, batch4, batch5, batch6];+ // We should have spilledlet metrics = exec.metrics().unwrap();assert!(- metrics.spill_count().unwrap() > input_partitions.len(),- "Expected spill_count > {} for order-preserving repartition, but got {:?}",- input_partitions.len(),- metrics.spill_count()- );- assert!(- metrics.spilled_bytes().unwrap()- > all_batches- .iter()- .map(|b| b.get_array_memory_size())- .sum::<usize>(),- "Expected spilled_bytes > {} for order-preserving repartition, got {}",- all_batches- .iter()- .map(|b| b.get_array_memory_size())- .sum::<usize>(),- metrics.spilled_bytes().unwrap()- );- assert!(- metrics.spilled_rows().unwrap()- >= all_batches.iter().map(|b| b.num_rows()).sum::<usize>(),- "Expected spilled_rows > {} for order-preserving repartition, got {}",- all_batches.iter().map(|b| b.num_rows()).sum::<usize>(),- metrics.spilled_rows().unwrap()+ metrics.spill_count().unwrap() > 0,+ "Expected spilling to occur for order-preserving repartition at this \+ memory limit. If this fails, the memory limit may need adjustment.");-Ok(())}diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rsindex 8129c3d8f6..ade24ff053 100644--- a/datafusion/physical-plan/src/sorts/streaming_merge.rs+++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs@@ -29,7 +29,7 @@ use arrow::array::*;use arrow::datatypes::{DataType, SchemaRef};use datafusion_common::human_readable_size;use datafusion_common::{Result, assert_or_internal_err, internal_err};-use datafusion_execution::disk_manager::RefCountedTempFile;+use datafusion_execution::SpillFile;use datafusion_execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation, UnboundedMemoryPool,};@@ -59,7 +59,7 @@ macro_rules! merge_helper {}pub struct SortedSpillFile {- pub file: RefCountedTempFile,+ pub file: Arc<dyn SpillFile>,/// how much memory the largest memory batch is takingpub max_record_batch_memory: usize,@@ -67,12 +67,19 @@ pub struct SortedSpillFile {impl std::fmt::Debug for SortedSpillFile {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- write!(- f,- "SortedSpillFile({:?}) takes {}",- self.file.path(),- human_readable_size(self.max_record_batch_memory)- )+ match self.file.path() {+ Some(path) => write!(+ f,+ "SortedSpillFile({:?}) takes {}",+ path,+ human_readable_size(self.max_record_batch_memory)+ ),+ None => write!(+ f,+ "SortedSpillFile(<custom_backend>) takes {}",+ human_readable_size(self.max_record_batch_memory)+ ),+ }}}diff --git a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs b/datafusion/physical-plan/src/spill/in_progress_spill_file.rsindex e0548bd5bf..71d7cce1bc 100644--- a/datafusion/physical-plan/src/spill/in_progress_spill_file.rs+++ b/datafusion/physical-plan/src/spill/in_progress_spill_file.rs@@ -22,7 +22,7 @@ use std::sync::Arc;use arrow::array::RecordBatch;use datafusion_common::exec_datafusion_err;-use datafusion_execution::disk_manager::RefCountedTempFile;+use datafusion_execution::spill_file::SpillFile;use super::{IPCStreamWriter, gc_view_arrays,@@ -37,13 +37,13 @@ pub struct InProgressSpillFile {/// Lazily initialized writerwriter: Option<IPCStreamWriter>,/// Lazily initialized in-progress file, it will be moved out when the `finish` method is invoked- in_progress_file: Option<RefCountedTempFile>,+ in_progress_file: Option<Arc<dyn SpillFile>>,}impl InProgressSpillFile {pub fn new(spill_writer: Arc<SpillManager>,- in_progress_file: RefCountedTempFile,+ in_progress_file: Arc<dyn SpillFile>,) -> Self {Self {spill_writer,@@ -79,40 +79,27 @@ impl InProgressSpillFile {// when they come from different branches of a UnionExec. The SpillManager's// schema represents the canonical schema that all batches should conform to.let schema = self.spill_writer.schema();- if let Some(in_progress_file) = &mut self.in_progress_file {+ if let Some(in_progress_file) = &self.in_progress_file {+ let spill_writer = in_progress_file.open_writer()?;+self.writer = Some(IPCStreamWriter::new(- in_progress_file.path(),+ spill_writer,schema.as_ref(),self.spill_writer.compression,)?);// Update metricsself.spill_writer.metrics.spill_file_count.add(1);-- // Update initial size (schema/header)- in_progress_file.update_disk_usage()?;- let initial_size = in_progress_file.current_disk_usage();- self.spill_writer- .metrics- .spilled_bytes- .add(initial_size as usize);+ let header_bytes = self.writer.as_ref().unwrap().bytes_written();+ self.spill_writer.metrics.spilled_bytes.add(header_bytes);}}if let Some(writer) = &mut self.writer {- let (spilled_rows, _) = writer.write(&gc_batch)?;- if let Some(in_progress_file) = &mut self.in_progress_file {- let pre_size = in_progress_file.current_disk_usage();- in_progress_file.update_disk_usage()?;- let post_size = in_progress_file.current_disk_usage();-- self.spill_writer.metrics.spilled_rows.add(spilled_rows);- self.spill_writer- .metrics- .spilled_bytes- .add((post_size - pre_size) as usize);- } else {- unreachable!() // Already checked inside current function- }+ // The writer calculates how many serialized bytes were emitted+ let (spilled_rows, delta_bytes) = writer.write(&gc_batch)?;++ self.spill_writer.metrics.spilled_rows.add(spilled_rows);+ self.spill_writer.metrics.spilled_bytes.add(delta_bytes);}gc_batch.get_sliced_size()}@@ -126,31 +113,26 @@ impl InProgressSpillFile {/// Returns a reference to the in-progress file, if it exists./// This can be used to get the file path for creating readers before the file is finished.- pub fn file(&self) -> Option<&RefCountedTempFile> {+ pub fn file(&self) -> Option<&Arc<dyn SpillFile>> {self.in_progress_file.as_ref()}- /// Finalizes the file, returning the completed file reference.+ /// Finalizes the write process, returning the completed `SpillFile`./// If there are no batches spilled before, it returns `None`.- pub fn finish(&mut self) -> Result<Option<RefCountedTempFile>> {- if let Some(writer) = &mut self.writer {- writer.finish()?;+ pub fn finish(&mut self) -> Result<Option<Arc<dyn SpillFile>>> {+ if self.in_progress_file.is_none() && self.writer.is_none() {+ return Err(exec_datafusion_err!(+ "Finish operation failed: file has already been finalized."+ ));+ }+ if let Some(mut writer) = self.writer.take() {+ // Finish the writer and capture any final trailing bytes emitted+ let delta_bytes = writer.finish()?;+ self.spill_writer.metrics.spilled_bytes.add(delta_bytes);} else {return Ok(None);}- // Since spill files are append-only, add the file size to spilled_bytes- if let Some(in_progress_file) = &mut self.in_progress_file {- // Since writer.finish() writes continuation marker and message length at the end- let pre_size = in_progress_file.current_disk_usage();- in_progress_file.update_disk_usage()?;- let post_size = in_progress_file.current_disk_usage();- self.spill_writer- .metrics- .spilled_bytes- .add((post_size - pre_size) as usize);- }-Ok(self.in_progress_file.take())}}diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rsindex 00c9ac0631..addcf78d2d 100644--- a/datafusion/physical-plan/src/spill/mod.rs+++ b/datafusion/physical-plan/src/spill/mod.rs@@ -21,16 +21,13 @@ pub(crate) mod in_progress_spill_file;pub(crate) mod replayable_spill_input;pub(crate) mod spill_manager;pub mod spill_pool;-+use datafusion_execution::spill_file::SpillWriter;// Moved for refactor, re-export to keep the public API stablepub use datafusion_common::utils::memory::get_record_batch_memory_size;// Re-export SpillManager for doctests only (hidden from public docs)#[doc(hidden)]pub use spill_manager::SpillManager;-use std::fs::File;-use std::io::BufReader;-use std::path::Path;use std::pin::Pin;use std::sync::Arc;use std::task::{Context, Poll};@@ -39,225 +36,216 @@ use arrow::array::{Array, ArrayRef, BinaryViewArray, BufferSpec, GenericByteViewArray, StringViewArray,layout, make_array,};+use arrow::buffer::Buffer;use arrow::datatypes::DataType;use arrow::datatypes::{ByteViewType, Schema, SchemaRef};use arrow::ipc::{MetadataVersion,- reader::StreamReader,+ reader::StreamDecoder,writer::{IpcWriteOptions, StreamWriter},};use arrow::record_batch::RecordBatch;use arrow_data::ArrayDataBuilder;use arrow_ipc::CompressionType;+use datafusion_common::Result;use datafusion_common::config::SpillCompression;-use datafusion_common::{DataFusionError, Result, exec_datafusion_err, exec_err};-use datafusion_common_runtime::SpawnedTask;use datafusion_execution::RecordBatchStream;-use datafusion_execution::disk_manager::RefCountedTempFile;-use futures::{FutureExt as _, Stream};+use datafusion_execution::spill_file::SpillFile;+use futures::Stream;use log::debug;-/// Stream that reads spill files from disk where each batch is read in a spawned blocking task-/// It will read one batch at a time and will not do any buffering, to buffer data use [`crate::common::spawn_buffered`]-///-/// A simpler solution would be spawning a long-running blocking task for each-/// file read (instead of each batch). This approach does not work because when-/// the number of concurrent reads exceeds the Tokio thread pool limit,-/// deadlocks can occur and block progress.+/// Stream that reads spill files from a [`SpillFile`] backend as a stream of [`RecordBatch`]es.+/// Uses [`StreamDecoder`] to decode IPC bytes received from the backend's async byte stream.+/// Backends handle their own threading concerns internally - OS files use+/// `tokio::fs::File` which performs blocking IO per-syscall without holding a thread+/// for the file's lifetime, avoiding deadlocks when concurrent reads exceed thread pool limits.struct SpillReaderStream {schema: SchemaRef,- state: SpillReaderStreamState,+ decoder: StreamDecoder,+ byte_stream: Pin<Box<dyn Stream<Item = Result<bytes::Bytes>> + Send>>,+ is_done: bool,+/// Maximum memory size observed among spilling sorted record batches./// This is used for validation purposes during reading each RecordBatch from spill./// For context on why this value is recorded and validated,/// see `physical_plan/sort/multi_level_merge.rs`.max_record_batch_memory: Option<usize>,-}--// Small margin allowed to accommodate slight memory accounting variation-const SPILL_BATCH_MEMORY_MARGIN: usize = 4096;--/// When we poll for the next batch, we will get back both the batch and the reader,-/// so we can call `next` again.-type NextRecordBatchResult = Result<(StreamReader<BufReader<File>>, Option<RecordBatch>)>;--enum SpillReaderStreamState {- /// Initial state: the stream was not initialized yet- /// and the file was not opened- Uninitialized(RefCountedTempFile),- /// A read is in progress in a spawned blocking task for which we hold the handle.- ReadInProgress(SpawnedTask<NextRecordBatchResult>),+ /// Holds leftover bytes from a chunk when a batch is yielded early+ current_buffer: Buffer,- /// A read has finished and we wait for being polled again in order to start reading the next batch.- Waiting(StreamReader<BufReader<File>>),+ /// Keeps the file alive until the stream is dropped+ _spill_file: Arc<dyn SpillFile>,- /// The stream has finished, successfully or not.- Done,+ schema_validated: bool,}+// Small margin allowed to accommodate slight memory accounting variation+const SPILL_BATCH_MEMORY_MARGIN: usize = 4096;+impl SpillReaderStream {fn new(schema: SchemaRef,- spill_file: RefCountedTempFile,+ spill_file: Arc<dyn SpillFile>,max_record_batch_memory: Option<usize>,- ) -> Self {- Self {+ ) -> Result<Self> {+ let byte_stream = spill_file.read_stream()?;+ // DataFusion controls what it writes so it can trust its own IPC output,+ // matching the behavior of the previous StreamReader-based implementation.+ let decoder = unsafe { StreamDecoder::new().with_skip_validation(true) };+ Ok(Self {schema,- state: SpillReaderStreamState::Uninitialized(spill_file),+ decoder,+ byte_stream,max_record_batch_memory,- }+ is_done: false,+ current_buffer: Buffer::from(&[]),+ _spill_file: spill_file,+ schema_validated: false,+ })}+}- fn poll_next_inner(- &mut self,- cx: &mut Context<'_>,- ) -> Poll<Option<Result<RecordBatch>>> {- match &mut self.state {- SpillReaderStreamState::Uninitialized(_) => {- // Temporarily replace with `Done` to be able to pass the file to the task.- let SpillReaderStreamState::Uninitialized(spill_file) =- std::mem::replace(&mut self.state, SpillReaderStreamState::Done)- else {- unreachable!()- };-- let expected_schema = Arc::clone(&self.schema);- let task = SpawnedTask::spawn_blocking(move || {- let file = BufReader::new(File::open(spill_file.path())?);- // SAFETY: DataFusion's spill writer strictly follows Arrow IPC specifications- // with validated schemas and buffers. Skip redundant validation during read- // to speedup read operation. This is safe for DataFusion as input guaranteed to be correct when written.- let mut reader = unsafe {- StreamReader::try_new(file, None)?.with_skip_validation(true)- };-- // Validate the schema read from Arrow IPC file is the same as the- // schema of the current `SpillManager`- let actual_schema = reader.schema();-- if actual_schema != expected_schema {- return exec_err!(- "Spill file schema mismatch: expected {}, got {}. \- The caller must use the same SpillManager that created the spill file to read it.",- expected_schema,- actual_schema- );- }-- // TODO: Same-schema reads from a different SpillManager still pass today.- // Add a SpillManager UID to IPC metadata and validate it here as well.- let next_batch = reader.next().transpose()?;-- Ok((reader, next_batch))- });+impl Stream for SpillReaderStream {+ type Item = Result<RecordBatch>;- self.state = SpillReaderStreamState::ReadInProgress(task);+ fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {+ let this = self.get_mut();- // Poll again immediately so the inner task is polled and the waker is- // registered.- self.poll_next_inner(cx)- }+ if this.is_done {+ return Poll::Ready(None);+ }- SpillReaderStreamState::ReadInProgress(task) => {- let result = futures::ready!(task.poll_unpin(cx))- .unwrap_or_else(|err| Err(DataFusionError::External(Box::new(err))));-- match result {- Ok((reader, batch)) => {- match batch {- Some(batch) => {- if let Some(max_record_batch_memory) =- self.max_record_batch_memory- {- let actual_size =- get_record_batch_memory_size(&batch);- if actual_size- > max_record_batch_memory- + SPILL_BATCH_MEMORY_MARGIN- {- debug!(- "Record batch memory usage ({actual_size} bytes) exceeds the expected limit ({max_record_batch_memory} bytes) \n\- by more than the allowed tolerance ({SPILL_BATCH_MEMORY_MARGIN} bytes).\n\- This likely indicates a bug in memory accounting during spilling.\n\- Please report this issue in https://github.com/apache/datafusion/issues/17340."- );- }- }- self.state = SpillReaderStreamState::Waiting(reader);-- Poll::Ready(Some(Ok(batch)))+ loop {+ if !this.current_buffer.is_empty() {+ match this.decoder.decode(&mut this.current_buffer) {+ Ok(Some(batch)) => {+ // One-time schema validation on the first decoded batch.+ // The IPC stream embeds the writer's schema in its header;+ // StreamDecoder surfaces it via the first batch's schema.+ // We check here rather than in new() because schema bytes+ // only arrive after decoding the IPC header from the stream.+ if !this.schema_validated {+ this.schema_validated = true;+ let actual = batch.schema();+ if actual != this.schema {+ this.is_done = true;+ return Poll::Ready(Some(Err(+ datafusion_common::exec_datafusion_err!(+ "Spill file schema mismatch: expected {}, got {}. \+ The caller must use the same SpillManager that created \+ the spill file to read it.",+ this.schema,+ actual+ ),+ )));}- None => {- // Stream is done- self.state = SpillReaderStreamState::Done;-- Poll::Ready(None)+ }+ if let Some(max_record_batch_memory) =+ this.max_record_batch_memory+ {+ let actual_size = get_record_batch_memory_size(&batch);+ if actual_size+ > max_record_batch_memory + SPILL_BATCH_MEMORY_MARGIN+ {+ debug!(+ "Record batch memory usage ({actual_size} bytes) exceeds the expected limit ({max_record_batch_memory} bytes) \n\+ by more than the allowed tolerance ({SPILL_BATCH_MEMORY_MARGIN} bytes).\n\+ This likely indicates a bug in memory accounting during spilling."+ );}}+ return Poll::Ready(Some(Ok(batch)));}- Err(err) => {- self.state = SpillReaderStreamState::Done;-- Poll::Ready(Some(Err(err)))+ Ok(None) => {+ // The chunk didn't form a complete message. Arrow consumed the partial bytes+ // into its internal scratch pad, leaving our current_buffer completely empty.+ // We do nothing and fall through to fetch more data.+ }+ Err(e) => {+ this.is_done = true;+ return Poll::Ready(Some(Err(e.into())));}}}- SpillReaderStreamState::Waiting(_) => {- // Temporarily replace with `Done` to be able to pass the file to the task.- let SpillReaderStreamState::Waiting(mut reader) =- std::mem::replace(&mut self.state, SpillReaderStreamState::Done)- else {- unreachable!()- };-- let task = SpawnedTask::spawn_blocking(move || {- let next_batch = reader.next().transpose()?;+ match futures::ready!(this.byte_stream.as_mut().poll_next(cx)) {+ Some(Ok(chunk)) => {+ this.current_buffer = Buffer::from(chunk);+ }+ Some(Err(e)) => {+ this.is_done = true;+ return Poll::Ready(Some(Err(e)));+ }+ None => {+ this.is_done = true;- Ok((reader, next_batch))- });+ if let Err(e) = this.decoder.finish() {+ return Poll::Ready(Some(Err(e.into())));+ }+ return Poll::Ready(None);+ }+ }+ }+ }+}- self.state = SpillReaderStreamState::ReadInProgress(task);+impl RecordBatchStream for SpillReaderStream {+ fn schema(&self) -> SchemaRef {+ Arc::clone(&self.schema)+ }+}- // Poll again immediately so the inner task is polled and the waker is- // registered.- self.poll_next_inner(cx)- }+/// A wrapper that counts the exact compressed IPC bytes written by Arrow.+///+/// Arrow's `StreamWriter` does not return the number of bytes written during its+/// `write()` calls. To accurately track the `spilled_bytes` metrics (especially+/// when LZ4/ZSTD compression is applied), we must intercept the `std::io::Write`+/// trait boundary to count the final serialized payload size.+pub(crate) struct TrackingSpillWriter {+ inner: Box<dyn SpillWriter>,+ pub(crate) total_bytes_written: usize,+}- SpillReaderStreamState::Done => Poll::Ready(None),+impl TrackingSpillWriter {+ pub fn new(inner: Box<dyn SpillWriter>) -> Self {+ Self {+ inner,+ total_bytes_written: 0,}}++ pub fn finish(mut self) -> Result<()> {+ self.inner.finish()+ }}-impl Stream for SpillReaderStream {- type Item = Result<RecordBatch>;+impl std::io::Write for TrackingSpillWriter {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ let n = self.inner.write(buf)?;- fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {- self.get_mut().poll_next_inner(cx)+ self.total_bytes_written += n;++ Ok(n)}-}-impl RecordBatchStream for SpillReaderStream {- fn schema(&self) -> SchemaRef {- Arc::clone(&self.schema)+ fn flush(&mut self) -> std::io::Result<()> {+ self.inner.flush()}}-/// Write in Arrow IPC Stream format to a file.-///-/// Stream format is used for spill because it supports dictionary replacement, and the random-/// access of IPC File format is not needed (IPC File format doesn't support dictionary replacement).+/// Write in Arrow IPC Stream format to an underlying `SpillWriter` backend.+/// Stream format also supports dictionary replacement.struct IPCStreamWriter {/// Inner writer- pub writer: StreamWriter<File>,+ writer: Option<StreamWriter<TrackingSpillWriter>>,/// Batches written- pub num_batches: usize,+ num_batches: usize,/// Rows written- pub num_rows: usize,+ num_rows: usize,/// Bytes written- pub num_bytes: usize,+ num_bytes: usize,}impl IPCStreamWriter {@@ -274,14 +262,10 @@ impl IPCStreamWriter {/// rather than relying solely on workspace-level feature unification;/// see #21917.pub fn new(- path: &Path,+ spill_writer: Box<dyn SpillWriter>,schema: &Schema,spill_compression: SpillCompression,) -> Result<Self> {- let file = File::create(path).map_err(|e| {- exec_datafusion_err!("(Hint: you may increase the file descriptor limit with shell command 'ulimit -n 4096') Failed to create partition file at {path:?}: {e:?}")- })?;-let metadata_version = MetadataVersion::V5;// Depending on the schema, some array types such as StringViewArray require larger (16 byte in this case) alignment.// If the actual buffer layout after IPC read does not satisfy the alignment requirement,@@ -291,15 +275,18 @@ impl IPCStreamWriter {let alignment = get_max_alignment_for_schema(schema);let mut write_options =IpcWriteOptions::try_new(alignment, false, metadata_version)?;+let compression_type = Option::<CompressionType>::from(spill_compression);write_options = write_options.try_with_compression(compression_type)?;- let writer = StreamWriter::try_new_with_options(file, schema, write_options)?;+ let adapter = TrackingSpillWriter::new(spill_writer);+ let writer = StreamWriter::try_new_with_options(adapter, schema, write_options)?;+Ok(Self {num_batches: 0,num_rows: 0,num_bytes: 0,- writer,+ writer: Some(writer),})}@@ -307,23 +294,50 @@ impl IPCStreamWriter {////// Returns a tuple containing the change in the number of rows and bytes written.pub fn write(&mut self, batch: &RecordBatch) -> Result<(usize, usize)> {- self.writer.write(batch)?;+ let writer = self.writer.as_mut().unwrap();++ let bytes_before = writer.get_ref().total_bytes_written;+ writer.write(batch)?;+ let bytes_after = writer.get_ref().total_bytes_written;self.num_batches += 1;let delta_num_rows = batch.num_rows();self.num_rows += delta_num_rows;- let delta_num_bytes: usize = batch.get_array_memory_size();+ let delta_num_bytes = bytes_after - bytes_before;self.num_bytes += delta_num_bytes;Ok((delta_num_rows, delta_num_bytes))}pub fn flush(&mut self) -> Result<()> {- self.writer.flush()?;+ use std::io::Write;+ if let Some(writer) = &mut self.writer {+ writer.get_mut().flush()?;+ }Ok(())}- /// Finish the writer- pub fn finish(&mut self) -> Result<()> {- self.writer.finish().map_err(Into::into)+ /// Finish the writer.+ ///+ /// Returns the number of trailing bytes written during the finish operation+ /// (e.g., IPC metadata and footers).+ pub fn finish(&mut self) -> Result<usize> {+ let mut writer = self.writer.take().unwrap();++ let bytes_before = writer.get_ref().total_bytes_written;+ writer.finish()?; // Writes IPC tail++ // Extract the adapter and flush the final bytes+ let adapter = writer.into_inner()?;+ let bytes_after = adapter.total_bytes_written;+ adapter.finish()?;++ Ok(bytes_after - bytes_before)+ }+ /// Returns the total number of bytes written so far+ pub fn bytes_written(&self) -> usize {+ self.writer+ .as_ref()+ .map(|w| w.get_ref().total_bytes_written)+ .unwrap_or(0)}}@@ -500,7 +514,7 @@ fn calculate_string_view_waste_ratio(array: &StringViewArray) -> f64 {#[cfg(test)]fn calculate_view_waste_ratio<F>(len: usize,- data_buffers: &[arrow::buffer::Buffer],+ data_buffers: &[Buffer],get_value_size: F,) -> f64where@@ -558,7 +572,7 @@ mod tests {let spill_file = spill_manager.spill_record_batch_and_finish(&[batch1, batch2], "Test")?.unwrap();- assert!(spill_file.path().exists());+ assert!(spill_file.path().unwrap().exists());let spilled_rows = spill_manager.metrics.spilled_rows.value();assert_eq!(spilled_rows, num_rows);@@ -655,7 +669,7 @@ mod tests {"Test Spill",)?.unwrap();- assert!(spill_file.path().exists());+ assert!(spill_file.path().unwrap().exists());assert!(max_batch_mem > 0);let stream = spill_manager.read_spill_as_stream(spill_file, None)?;@@ -685,7 +699,7 @@ mod tests {async fn validate(spill_manager: &SpillManager,- spill_file: RefCountedTempFile,+ spill_file: Arc<dyn SpillFile>,num_rows: usize,schema: SchemaRef,batch_count: usize,@@ -735,14 +749,14 @@ mod tests {let zstd_spill_file = zstd_spill_manager.spill_record_batch_and_finish(&batches, "ZSTD_Test")?.unwrap();- assert!(uncompressed_spill_file.path().exists());- assert!(lz4_spill_file.path().exists());- assert!(zstd_spill_file.path().exists());+ assert!(uncompressed_spill_file.path().unwrap().exists());+ assert!(lz4_spill_file.path().unwrap().exists());+ assert!(zstd_spill_file.path().unwrap().exists());- let lz4_spill_size = std::fs::metadata(lz4_spill_file.path())?.len();- let zstd_spill_size = std::fs::metadata(zstd_spill_file.path())?.len();+ let lz4_spill_size = std::fs::metadata(lz4_spill_file.path().unwrap())?.len();+ let zstd_spill_size = std::fs::metadata(zstd_spill_file.path().unwrap())?.len();let uncompressed_spill_size =- std::fs::metadata(uncompressed_spill_file.path())?.len();+ std::fs::metadata(uncompressed_spill_file.path().unwrap())?.len();assert!(uncompressed_spill_size > lz4_spill_size);assert!(uncompressed_spill_size > zstd_spill_size);@@ -797,7 +811,7 @@ mod tests {let temp_file = spill_manager.spill_record_batch_and_finish(&[batch], "Test")?;assert!(temp_file.is_some());- assert!(temp_file.unwrap().path().exists());+ assert!(temp_file.unwrap().path().unwrap().exists());Ok(())}@@ -871,7 +885,7 @@ mod tests {let completed_file = in_progress_file.finish()?;assert!(completed_file.is_some());- assert!(completed_file.unwrap().path().exists());+ assert!(completed_file.unwrap().path().unwrap().exists());verify_metrics(&in_progress_file, 1, 712, 6)?;// Double finish produce errorlet result = in_progress_file.finish();@@ -1285,7 +1299,7 @@ mod tests {}let spill_file = in_progress_file.finish()?.unwrap();- let file_size = fs::metadata(spill_file.path())?.len() as usize;+ let file_size = fs::metadata(spill_file.path().unwrap())?.len() as usize;let theoretical_without_gc = total_buffer_size * sliced_batches.len();let reduction_percent = ((theoretical_without_gc - file_size) as f64@@ -1455,7 +1469,7 @@ mod tests {.unwrap();// 4. Check file size on disk- let file_size = fs::metadata(spill_file.path())?.len();+ let file_size = fs::metadata(spill_file.path().unwrap())?.len();// The original buffer size is around 70KB.// Without GC, the spill file would be > 70KB.@@ -1499,7 +1513,7 @@ mod tests {.unwrap();// 4. Check file size on disk- let file_size = fs::metadata(spill_file.path())?.len();+ let file_size = fs::metadata(spill_file.path().unwrap())?.len();// Original buffer is 100KB.// With GC, it should be much smaller.diff --git a/datafusion/physical-plan/src/spill/replayable_spill_input.rs b/datafusion/physical-plan/src/spill/replayable_spill_input.rsindex fea998d268..94a0aef7dc 100644--- a/datafusion/physical-plan/src/spill/replayable_spill_input.rs+++ b/datafusion/physical-plan/src/spill/replayable_spill_input.rs@@ -26,9 +26,8 @@ use std::task::{Context, Poll};use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::{Result, internal_err};-use datafusion_execution::RecordBatchStream;use datafusion_execution::SendableRecordBatchStream;-use datafusion_execution::disk_manager::RefCountedTempFile;+use datafusion_execution::{RecordBatchStream, SpillFile};use futures::Stream;use parking_lot::Mutex;@@ -91,7 +90,7 @@ pub(crate) struct ReplayableStreamSource {/// Inner state exclusively owned by either [`ReplayableStreamSource`] or one [`ReplayableSpillStream`]enum StateInner {Unopened,- Replayable(Option<RefCountedTempFile>),+ Replayable(Option<Arc<dyn SpillFile>>),Poisoned,}@@ -222,10 +221,10 @@ impl ReplayableSpillStream {schema: SchemaRef,spill_manager: &SpillManager,shared_state: Arc<Mutex<Option<StateInner>>>,- spill_file: Option<RefCountedTempFile>,+ spill_file: Option<Arc<dyn SpillFile>>,) -> Result<Self> {let inner = if let Some(file) = spill_file.as_ref() {- spill_manager.read_spill_as_stream(file.clone(), None)?+ spill_manager.read_spill_as_stream(Arc::clone(file), None)?} else {Box::pin(EmptyRecordBatchStream::new(Arc::clone(&schema)))};diff --git a/datafusion/physical-plan/src/spill/spill_manager.rs b/datafusion/physical-plan/src/spill/spill_manager.rsindex 365a9f977e..3f305c16a6 100644--- a/datafusion/physical-plan/src/spill/spill_manager.rs+++ b/datafusion/physical-plan/src/spill/spill_manager.rs@@ -25,8 +25,8 @@ use arrow::datatypes::{ByteViewType, SchemaRef};use arrow::record_batch::RecordBatch;use datafusion_common::{DataFusionError, Result, config::SpillCompression};use datafusion_execution::SendableRecordBatchStream;-use datafusion_execution::disk_manager::RefCountedTempFile;use datafusion_execution::runtime_env::RuntimeEnv;+use datafusion_execution::spill_file::SpillFile;use std::borrow::Borrow;use std::sync::Arc;@@ -99,7 +99,7 @@ impl SpillManager {&self,batches: &[RecordBatch],request_msg: &str,- ) -> Result<Option<RefCountedTempFile>> {+ ) -> Result<Option<Arc<dyn SpillFile>>> {let mut in_progress_file = self.create_in_progress_file(request_msg)?;for batch in batches {@@ -115,7 +115,7 @@ impl SpillManager {&self,mut iter: impl Iterator<Item = Result<impl Borrow<RecordBatch>>>,request_description: &str,- ) -> Result<Option<(RefCountedTempFile, usize)>> {+ ) -> Result<Option<(Arc<dyn SpillFile>, usize)>> {let mut in_progress_file = self.create_in_progress_file(request_description)?;let mut max_record_batch_size = 0;@@ -141,7 +141,7 @@ impl SpillManager {&self,stream: &mut SendableRecordBatchStream,request_description: &str,- ) -> Result<Option<(RefCountedTempFile, usize)>> {+ ) -> Result<Option<(Arc<dyn SpillFile>, usize)>> {use futures::StreamExt;let mut in_progress_file = self.create_in_progress_file(request_description)?;@@ -178,14 +178,14 @@ impl SpillManager {/// the merge degree when merging multiple sorted runs.pub fn read_spill_as_stream(&self,- spill_file_path: RefCountedTempFile,+ spill_file_path: Arc<dyn SpillFile>,max_record_batch_memory: Option<usize>,) -> Result<SendableRecordBatchStream> {let stream = Box::pin(cooperative(SpillReaderStream::new(Arc::clone(&self.schema),spill_file_path,max_record_batch_memory,- )));+ )?));Ok(spawn_buffered(stream, self.batch_read_buffer_capacity))}@@ -193,14 +193,14 @@ impl SpillManager {/// Same as `read_spill_as_stream`, but without buffering.pub fn read_spill_as_stream_unbuffered(&self,- spill_file_path: RefCountedTempFile,+ spill_file_path: Arc<dyn SpillFile>,max_record_batch_memory: Option<usize>,) -> Result<SendableRecordBatchStream> {Ok(Box::pin(cooperative(SpillReaderStream::new(Arc::clone(&self.schema),spill_file_path,max_record_batch_memory,- ))))+ )?)))}}diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rsindex 2639188a26..75da18315f 100644--- a/datafusion/physical-plan/src/spill/spill_pool.rs+++ b/datafusion/physical-plan/src/spill/spill_pool.rs@@ -25,8 +25,7 @@ use parking_lot::Mutex;use arrow::datatypes::SchemaRef;use arrow::record_batch::RecordBatch;use datafusion_common::Result;-use datafusion_execution::disk_manager::RefCountedTempFile;-use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream};+use datafusion_execution::{RecordBatchStream, SendableRecordBatchStream, SpillFile};use super::in_progress_spill_file::InProgressSpillFile;use super::spill_manager::SpillManager;@@ -179,7 +178,9 @@ impl SpillPoolWriter {let writer = spill_manager.create_in_progress_file("SpillPool")?;// Clone the file so readers can access it immediately- let file = writer.file().expect("InProgressSpillFile should always have a file when it is first created").clone();+ let file = Arc::clone(writer.file().expect(+ "InProgressSpillFile should always have a file when it is first created",+ ));let file_shared = Arc::new(Mutex::new(ActiveSpillFileShared {writer: Some(writer),@@ -482,7 +483,7 @@ struct ActiveSpillFileShared {writer: Option<InProgressSpillFile>,/// The spill file, set when the writer finishes./// Taken by the reader when creating a stream (the file stays open via file handles).- file: Option<RefCountedTempFile>,+ file: Option<Arc<dyn SpillFile>>,/// Total number of batches written to this filebatches_written: usize,/// Estimated size in bytes of data written to this file@@ -507,25 +508,25 @@ impl ActiveSpillFileShared {}}-/// Reader state for a SpillFile (owned by individual SpillFile instances).+/// Reader state for a SpillPoolFile (owned by individual SpillPoolFile instances)./// This is kept separate from the shared state to avoid holding locks during I/O.-struct SpillFileReader {+struct SpillPoolFileReader {/// The actual stream reading from diskstream: SendableRecordBatchStream,/// Number of batches this reader has consumedbatches_read: usize,}-struct SpillFile {+struct SpillPoolFile {/// Shared coordination state (contains writer and batch counts)shared: Arc<Mutex<ActiveSpillFileShared>>,- /// Reader state (lazy-initialized, owned by this SpillFile)- reader: Option<SpillFileReader>,+ /// Reader state (lazy-initialized, owned by this SpillPoolFile)+ reader: Option<SpillPoolFileReader>,/// Spill manager for creating readersspill_manager: Arc<SpillManager>,}-impl Stream for SpillFile {+impl Stream for SpillPoolFile {type Item = Result<RecordBatch>;fn poll_next(@@ -568,7 +569,7 @@ impl Stream for SpillFile {.read_spill_as_stream_unbuffered(file, None){Ok(stream) => {- self.reader = Some(SpillFileReader {+ self.reader = Some(SpillPoolFileReader {stream,batches_read: 0,});@@ -627,8 +628,8 @@ impl Stream for SpillFile {pub struct SpillPoolReader {/// Shared reference to the spill poolshared: Arc<Mutex<SpillPoolShared>>,- /// Current SpillFile we're reading from- current_file: Option<SpillFile>,+ /// Current SpillPoolFile we're reading from+ current_file: Option<SpillPoolFile>,/// Schema of the spilled dataschema: SchemaRef,}@@ -706,12 +707,12 @@ impl Stream for SpillPoolReader {// Peek at the front of the queue (don't pop yet)if let Some(file_shared) = shared.files.front() {- // Create a SpillFile from the shared state+ // Create a SpillPoolFile from the shared statelet spill_manager = Arc::clone(&shared.spill_manager);let file_shared = Arc::clone(file_shared);- drop(shared); // Release lock before creating SpillFile+ drop(shared); // Release lock before creating SpillPoolFile- self.current_file = Some(SpillFile {+ self.current_file = Some(SpillPoolFile {shared: file_shared,reader: None,spill_manager,@@ -1463,7 +1464,7 @@ mod tests {let schema = create_test_schema();let spill_manager = Arc::new(SpillManager::new(runtime, metrics.clone(), schema));- let (writer, mut reader) = channel(batch_size, spill_manager);+ let (writer, mut reader) = channel(batch_size - 1, spill_manager);// Step 3: Write NUM_BATCHES batches to create approximately NUM_BATCHES filesfor i in 0..NUM_BATCHES {@@ -1474,10 +1475,8 @@ mod tests {// Check how many files were created (should be at least a few due to file rotation)let file_count = metrics.spill_file_count.value();assert_eq!(- file_count,- NUM_BATCHES - 1,- "Expected at {} files with rotation, got {file_count}",- NUM_BATCHES - 1+ file_count, NUM_BATCHES,+ "Expected at {NUM_BATCHES} files with rotation, got {file_count}");// Step 4: Verify initial disk usage reflects all filesdiff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.mdindex d0778a3619..307de722bd 100644--- a/docs/source/library-user-guide/upgrading/55.0.0.md+++ b/docs/source/library-user-guide/upgrading/55.0.0.md@@ -25,6 +25,35 @@in this section pertains to features and changes that have already been mergedto the main branch and are awaiting release in this version.+### User `SpillFile` traits instead of [`RefCountedTempFile`]++Spill file APIs now use the `datafusion_execution::SpillFile` trait instead of+the concrete [`RefCountedTempFile`] type. [`DiskManager::create_tmp_file`] now+returns `Arc<dyn SpillFile>`.+This change was introduced in [PR #21882], which adds pluggable spill file+backends via `SpillFile` and `TempFileFactory`.++If your code matched on [`DiskManagerMode`], add a `DiskManagerMode::Custom(_)`+arm.++If your code wrote directly to a [`RefCountedTempFile`] or called+[`RefCountedTempFile::update_disk_usage`], open a spill writer instead:++```diff+- temp_file.inner().as_file().write_all(bytes)?;+- temp_file.update_disk_usage()?;++ temp_file.open_writer()?.write_all(bytes)?;+```++Use `temp_file.size()` instead of [`RefCountedTempFile::current_disk_usage`].++[`diskmanager::create_tmp_file`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.DiskManager.html#method.create_tmp_file+[`diskmanagermode`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/enum.DiskManagerMode.html+[`pr #21882`]: https://github.com/apache/datafusion/pull/21882+[`refcountedtempfile`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.RefCountedTempFile.html+[`refcountedtempfile::current_disk_usage`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.RefCountedTempFile.html#method.current_disk_usage+[`refcountedtempfile::update_disk_usage`]: https://docs.rs/datafusion-execution/latest/datafusion_execution/disk_manager/struct.RefCountedTempFile.html#method.update_disk_usage+### `Dialect::AVAILABLE` replaced by `Dialect::available()``datafusion_common::config::Dialect::AVAILABLE` has been removed. Use