Skip to content

Apache DataFusion SQL Query Engine

Make DiskManager max_temp_directory_size dynamically adjustable (#22246)

Bukhtawar Khan65f0c820bfda
diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs
index 1a14bd239a..070ea53343 100644
--- a/datafusion/execution/src/disk_manager.rs
+++ b/datafusion/execution/src/disk_manager.rs
@@ -75,7 +75,7 @@ impl DiskManagerBuilder {
match self.mode {
DiskManagerMode::OsTmpDirectory => Ok(DiskManager {
local_dirs: Mutex::new(Some(vec![])),
- max_temp_directory_size: self.max_temp_directory_size,
+ max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
}),
@@ -86,14 +86,14 @@ impl DiskManagerBuilder {
);
Ok(DiskManager {
local_dirs: Mutex::new(Some(local_dirs)),
- max_temp_directory_size: self.max_temp_directory_size,
+ max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
})
}
DiskManagerMode::Disabled => Ok(DiskManager {
local_dirs: Mutex::new(None),
- max_temp_directory_size: self.max_temp_directory_size,
+ max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
}),
@@ -167,8 +167,9 @@ pub struct DiskManager {
/// If `None` an error will be returned (configured not to spill)
local_dirs: Mutex<Option<Vec<Arc<TempDir>>>>,
/// The maximum amount of data (in bytes) stored inside the temporary directories.
- /// Default to 100GB
- max_temp_directory_size: u64,
+ /// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime
+ /// without requiring exclusive (`&mut`) access to the `DiskManager`.
+ max_temp_directory_size: AtomicU64,
/// Used disk space in the temporary directories. Now only spilled data for
/// external executors are counted.
used_disk_space: Arc<AtomicU64>,
@@ -199,7 +200,7 @@ impl DiskManager {
DiskManagerConfig::Existing(manager) => Ok(manager),
DiskManagerConfig::NewOs => Ok(Arc::new(Self {
local_dirs: Mutex::new(Some(vec![])),
- max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
+ max_temp_directory_size: AtomicU64::new(DEFAULT_MAX_TEMP_DIRECTORY_SIZE),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
})),
@@ -210,50 +211,57 @@ impl DiskManager {
);
Ok(Arc::new(Self {
local_dirs: Mutex::new(Some(local_dirs)),
- max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
+ max_temp_directory_size: AtomicU64::new(
+ DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
+ ),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
}))
}
DiskManagerConfig::Disabled => Ok(Arc::new(Self {
local_dirs: Mutex::new(None),
- max_temp_directory_size: DEFAULT_MAX_TEMP_DIRECTORY_SIZE,
+ max_temp_directory_size: AtomicU64::new(DEFAULT_MAX_TEMP_DIRECTORY_SIZE),
used_disk_space: Arc::new(AtomicU64::new(0)),
active_files_count: Arc::new(AtomicUsize::new(0)),
})),
}
}
+ /// Atomically set the max temp directory size at runtime.
+ ///
+ /// Takes `&self`, so it works through `Arc<DiskManager>` without requiring
+ /// exclusive access. Takes effect immediately for subsequent spill writes.
+ ///
+ /// Use this when you need to adjust the limit dynamically while queries
+ /// are running (e.g., adapting to available disk space).
pub fn set_max_temp_directory_size(
- &mut self,
+ &self,
max_temp_directory_size: u64,
) -> Result<()> {
- // If the disk manager is disabled and `max_temp_directory_size` is not 0,
- // this operation is not meaningful, fail early.
if self.local_dirs.lock().is_none() && max_temp_directory_size != 0 {
return config_err!(
"Cannot set max temp directory size for a disk manager that spilling is disabled"
);
}
- self.max_temp_directory_size = max_temp_directory_size;
+ self.max_temp_directory_size
+ .store(max_temp_directory_size, Ordering::Relaxed);
Ok(())
}
+ #[deprecated(
+ since = "54.0.0",
+ note = "Use `set_max_temp_directory_size` directly, it now takes &self"
+ )]
pub fn set_arc_max_temp_directory_size(
- this: &mut Arc<Self>,
+ this: &Arc<Self>,
max_temp_directory_size: u64,
) -> Result<()> {
- if let Some(inner) = Arc::get_mut(this) {
- inner.set_max_temp_directory_size(max_temp_directory_size)?;
- Ok(())
- } else {
- config_err!("DiskManager should be a single instance")
- }
+ this.set_max_temp_directory_size(max_temp_directory_size)
}
pub fn with_max_temp_directory_size(
- mut self,
+ self,
max_temp_directory_size: u64,
) -> Result<Self> {
self.set_max_temp_directory_size(max_temp_directory_size)?;
@@ -266,7 +274,7 @@ impl DiskManager {
/// Returns the maximum temporary directory size in bytes
pub fn max_temp_directory_size(&self) -> u64 {
- self.max_temp_directory_size
+ self.max_temp_directory_size.load(Ordering::Relaxed)
}
/// Returns the current spilling progress
@@ -418,11 +426,24 @@ impl RefCountedTempFile {
// 3. Check if the updated global disk usage exceeds the configured limit
let global_disk_usage = self.disk_manager.used_disk_space.load(Ordering::Relaxed);
- if global_disk_usage > self.disk_manager.max_temp_directory_size {
+ let limit = self
+ .disk_manager
+ .max_temp_directory_size
+ .load(Ordering::Relaxed);
+ if global_disk_usage > limit {
+ // Roll back: restore global counter to previous state so that
+ // Drop (which subtracts current_file_disk_usage = old value) remains
+ // consistent. Without this, the delta (new - old) leaks permanently.
+ self.disk_manager
+ .used_disk_space
+ .fetch_sub(new_disk_usage, Ordering::Relaxed);
+ self.disk_manager
+ .used_disk_space
+ .fetch_add(old_disk_usage, Ordering::Relaxed);
return resources_err!(
"The used disk space during the spilling process has exceeded the allowable limit of {}. \
Please try increasing the config: `datafusion.runtime.max_temp_directory_size`.",
- human_readable_size(self.disk_manager.max_temp_directory_size as usize)
+ human_readable_size(limit as usize)
);
}
@@ -796,4 +817,265 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_dynamic_limit_adjustment_through_shared_ref() -> Result<()> {
+ // Verify that set_max_temp_directory_size works through &self (not &mut self).
+ // This is the key behavioral change: the limit can be adjusted at runtime
+ // without exclusive access, enabling dynamic resize while queries are running.
+ let dm = DiskManager::builder()
+ .with_max_temp_directory_size(1024)
+ .build()?;
+ let dm = Arc::new(dm);
+
+ assert_eq!(dm.max_temp_directory_size(), 1024);
+
+ // Adjust through shared reference (simulates concurrent access via Arc)
+ dm.set_max_temp_directory_size(2048)?;
+ assert_eq!(dm.max_temp_directory_size(), 2048);
+
+ // Can also decrease
+ dm.set_max_temp_directory_size(512)?;
+ assert_eq!(dm.max_temp_directory_size(), 512);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dynamic_limit_concurrent_access() -> Result<()> {
+ // Verify that multiple threads can read and write the limit concurrently
+ let dm = Arc::new(
+ DiskManager::builder()
+ .with_max_temp_directory_size(1000)
+ .build()?,
+ );
+
+ let handles: Vec<_> = (0..8)
+ .map(|i| {
+ let dm = Arc::clone(&dm);
+ std::thread::spawn(move || {
+ // Each thread sets a different limit and reads it back
+ let new_limit = (i + 1) * 1000;
+ dm.set_max_temp_directory_size(new_limit).unwrap();
+ // Read should return SOME value set by one of the threads
+ let current = dm.max_temp_directory_size();
+ assert!((1000..=8000).contains(&current));
+ })
+ })
+ .collect();
+
+ for h in handles {
+ h.join().unwrap();
+ }
+
+ // Final value should be one of the values set by threads
+ let final_val = dm.max_temp_directory_size();
+ assert!((1000..=8000).contains(&final_val));
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_disabled_disk_manager_rejects_nonzero_limit() -> Result<()> {
+ let dm = DiskManager::builder()
+ .with_mode(DiskManagerMode::Disabled)
+ .build()?;
+ let dm = Arc::new(dm);
+
+ // Setting non-zero limit on disabled DiskManager should error
+ let result = dm.set_max_temp_directory_size(1024);
+ assert!(result.is_err());
+
+ // Setting zero is OK
+ assert!(dm.set_max_temp_directory_size(0).is_ok());
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_limit_decrease_below_current_usage() -> Result<()> {
+ // Scenario: DiskManager has 100GB limit, currently using 80GB.
+ // Admin lowers limit to 60GB. What happens?
+ //
+ // Expected behavior:
+ // - Existing spill files remain on disk (not deleted)
+ // - used_disk_space still reports 80GB
+ // - New spill writes FAIL immediately (80GB > 60GB new limit)
+ // - Once old queries complete and release their files (used drops below 60GB),
+ // new spill writes succeed again
+ //
+ // This demonstrates graceful degradation: lowering the limit doesn't
+ // reclaim existing files (would break running queries), but prevents
+ // additional spilling until usage drops naturally.
+ let dm = DiskManager::builder()
+ .with_max_temp_directory_size(100 * 1024 * 1024 * 1024) // 100GB
+ .build()?;
+ let dm = Arc::new(dm);
+
+ // Simulate 80GB of existing spill usage
+ dm.used_disk_space
+ .store(80 * 1024 * 1024 * 1024, Ordering::Relaxed);
+
+ assert_eq!(dm.max_temp_directory_size(), 100 * 1024 * 1024 * 1024);
+ assert_eq!(dm.used_disk_space(), 80 * 1024 * 1024 * 1024);
+
+ // Lower the limit to 60GB (below current usage)
+ dm.set_max_temp_directory_size(60 * 1024 * 1024 * 1024)?;
+ assert_eq!(dm.max_temp_directory_size(), 60 * 1024 * 1024 * 1024);
+
+ // Current usage (80GB) now exceeds the new limit (60GB).
+ // The used_disk_space is NOT reclaimed — existing files stay.
+ assert_eq!(dm.used_disk_space(), 80 * 1024 * 1024 * 1024);
+
+ // Any attempt to write MORE would be rejected at the SpillWriter level
+ // because used_disk_space(80GB) > max_temp_directory_size(60GB).
+ // (SpillWriter check: `global_disk_usage > limit` returns ResourcesExhausted)
+
+ // Simulate old queries completing: usage drops to 50GB
+ dm.used_disk_space
+ .store(50 * 1024 * 1024 * 1024, Ordering::Relaxed);
+
+ // Now usage (50GB) < limit (60GB) — new spill writes would succeed again
+ assert!(dm.used_disk_space() < dm.max_temp_directory_size());
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_limit_decrease_with_concurrent_queries() -> Result<()> {
+ // Scenario: Multiple threads spilling while limit is lowered concurrently.
+ // Demonstrates that:
+ // 1. In-flight spills that started before the limit change complete normally
+ // (they already incremented used_disk_space)
+ // 2. New spills after the limit change respect the new lower limit
+ // 3. No data corruption or panics from concurrent access
+ let dm = Arc::new(
+ DiskManager::builder()
+ .with_max_temp_directory_size(100 * 1024 * 1024) // 100MB
+ .build()?,
+ );
+
+ let barrier = Arc::new(std::sync::Barrier::new(5));
+
+ // 4 threads simulate concurrent spilling
+ let spill_handles: Vec<_> = (0..4)
+ .map(|_| {
+ let dm = Arc::clone(&dm);
+ let barrier = Arc::clone(&barrier);
+ std::thread::spawn(move || {
+ barrier.wait();
+ // Simulate spill: increment used_disk_space
+ dm.used_disk_space
+ .fetch_add(10 * 1024 * 1024, Ordering::Relaxed);
+ std::thread::sleep(std::time::Duration::from_millis(10));
+ // Simulate cleanup
+ dm.used_disk_space
+ .fetch_sub(10 * 1024 * 1024, Ordering::Relaxed);
+ })
+ })
+ .collect();
+
+ // 1 thread lowers the limit mid-flight
+ let dm_resize = Arc::clone(&dm);
+ let resize_barrier = Arc::clone(&barrier);
+ let resize_handle = std::thread::spawn(move || {
+ resize_barrier.wait();
+ // Lower limit while spills are in progress
+ dm_resize
+ .set_max_temp_directory_size(30 * 1024 * 1024) // 30MB
+ .unwrap();
+ });
+
+ for h in spill_handles {
+ h.join().unwrap();
+ }
+ resize_handle.join().unwrap();
+
+ // After all threads complete:
+ // - Limit is 30MB (last set by resize thread)
+ // - used_disk_space is 0 (all spills cleaned up)
+ // - No panics, no corruption
+ assert_eq!(dm.max_temp_directory_size(), 30 * 1024 * 1024);
+ assert_eq!(dm.used_disk_space(), 0);
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_rollback_on_limit_exceeded_then_drop_returns_to_zero() -> Result<()> {
+ // This test verifies that lowering the limit, failing a spill write,
+ // and then dropping the file leaves used_disk_space at zero.
+ //
+ // Without the rollback fix, the global counter would be permanently
+ // inflated by the delta between the new and old file sizes.
+ use std::fs::OpenOptions;
+ use std::io::Write;
+
+ let dm = Arc::new(
+ DiskManager::builder()
+ .with_max_temp_directory_size(10 * 1024 * 1024) // 10MB
+ .build()?,
+ );
+
+ // Create a temp file and write some data via a separate writable handle
+ let mut file = dm.create_tmp_file("test_rollback")?;
+ {
+ let path = file.path().to_path_buf();
+ let mut f = OpenOptions::new().append(true).open(&path)?;
+ let data = vec![0u8; 1024]; // 1KB
+ f.write_all(&data)?;
+ f.sync_all()?;
+ }
+ // Record the file's disk usage
+ file.update_disk_usage()?;
+ let usage_after_first_write = dm.used_disk_space();
+ assert!(usage_after_first_write > 0);
+
+ // Write more data to grow the file
+ {
+ let path = file.path().to_path_buf();
+ let mut f = OpenOptions::new().append(true).open(&path)?;
+ let data = vec![0u8; 4 * 1024]; // 4KB more
+ f.write_all(&data)?;
+ f.sync_all()?;
+ }
+ // Update disk usage — should succeed (still under 10MB)
+ file.update_disk_usage()?;
+ let usage_after_second_write = dm.used_disk_space();
+ assert!(usage_after_second_write > usage_after_first_write);
+
+ // Now lower the limit to 1 byte — below current usage
+ dm.set_max_temp_directory_size(1)?;
+
+ // Write even more data
+ {
+ let path = file.path().to_path_buf();
+ let mut f = OpenOptions::new().append(true).open(&path)?;
+ let data = vec![0u8; 2 * 1024]; // 2KB more
+ f.write_all(&data)?;
+ f.sync_all()?;
+ }
+
+ // This update_disk_usage should FAIL (exceeds new 1-byte limit)
+ let result = file.update_disk_usage();
+ assert!(result.is_err());
+ assert!(
+ result
+ .unwrap_err()
+ .to_string()
+ .contains("exceeded the allowable limit")
+ );
+
+ // Critical check: used_disk_space should still equal the LAST
+ // successful update (before the failed one), not be inflated
+ assert_eq!(dm.used_disk_space(), usage_after_second_write);
+
+ // Drop the file — should subtract the last successful file size
+ drop(file);
+
+ // After drop: used_disk_space must be zero (no leak)
+ assert_eq!(dm.used_disk_space(), 0);
+
+ Ok(())
+ }
}