Apache DataFusion SQL Query Engine
perf: create DiskManager Directories-mode working dirs lazily (#26583)
RAGHVENDRA KUMAR YADAV6ab30359a3fd
diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rsindex 5a883e8884..5dc07d6dea 100644--- a/datafusion/execution/src/disk_manager.rs+++ b/datafusion/execution/src/disk_manager.rs@@ -117,28 +117,28 @@ impl DiskManagerBuilder {match self.mode {DiskManagerMode::OsTmpDirectory => Ok(DiskManager {local_dirs: Mutex::new(Some(vec![])),+ conf_dirs: vec![],+ max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),+ max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),+ used_disk_space: Arc::new(AtomicU64::new(0)),+ active_files_count: Arc::new(AtomicUsize::new(0)),+ factory: None,+ }),+ DiskManagerMode::Directories(conf_dirs) => Ok(DiskManager {+ // Created on the first `create_tmp_file`, as `OsTmpDirectory`+ // already does, so building a `DiskManager` that never spills+ // costs no directory syscalls.+ local_dirs: Mutex::new(Some(vec![])),+ conf_dirs,max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),active_files_count: Arc::new(AtomicUsize::new(0)),factory: None,}),- DiskManagerMode::Directories(conf_dirs) => {- let local_dirs = create_local_dirs(&conf_dirs)?;- debug!(- "Created local dirs {local_dirs:?} as DataFusion working directory"- );- Ok(DiskManager {- local_dirs: Mutex::new(Some(local_dirs)),- max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),- max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),- used_disk_space: Arc::new(AtomicU64::new(0)),- active_files_count: Arc::new(AtomicUsize::new(0)),- factory: None,- })- }DiskManagerMode::Disabled => Ok(DiskManager {local_dirs: Mutex::new(None),+ conf_dirs: vec![],max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),@@ -147,6 +147,7 @@ impl DiskManagerBuilder {}),DiskManagerMode::Custom(factory) => Ok(DiskManager {local_dirs: Mutex::new(None),+ conf_dirs: vec![],max_temp_directory_size: AtomicU64::new(self.max_temp_directory_size),max_spill_merge_fan_in: AtomicUsize::new(self.max_spill_merge_fan_in),used_disk_space: Arc::new(AtomicU64::new(0)),@@ -195,6 +196,10 @@ pub struct DiskManager {/// If `Some(vec![])` a new OS specified temporary directory will be created/// If `None` an error will be returned (configured not to spill)local_dirs: Mutex<Option<Vec<Arc<TempDir>>>>,+ /// Directories configured by [`DiskManagerMode::Directories`], whose working+ /// directories are created on the first `create_tmp_file`. Empty in every+ /// other mode, where an OS temporary directory is used instead.+ conf_dirs: Vec<PathBuf>,/// The maximum amount of data (in bytes) stored inside the temporary directories./// Default to 100GB. Stored as `AtomicU64` so it can be adjusted at runtime/// without requiring exclusive (`&mut`) access to the `DiskManager`.@@ -354,17 +359,23 @@ impl DiskManager {)})?;- // Create a temporary directory if needed+ // Create the working directories if neededif local_dirs.is_empty() {- let tempdir = tempfile::tempdir().map_err(DataFusionError::IoError)?;+ if self.conf_dirs.is_empty() {+ let tempdir = tempfile::tempdir().map_err(DataFusionError::IoError)?;- debug!(- "Created directory '{:?}' as DataFusion tempfile directory for {}",- tempdir.path().to_string_lossy(),- request_description,- );+ debug!(+ "Created directory '{:?}' as DataFusion tempfile directory for {}",+ tempdir.path().to_string_lossy(),+ request_description,+ );- local_dirs.push(Arc::new(tempdir));+ local_dirs.push(Arc::new(tempdir));+ } else {+ let created = create_local_dirs(&self.conf_dirs)?;+ debug!("Created local dirs {created:?} as DataFusion working directory");+ local_dirs.extend(created);+ }}let dir_index = rng().random_range(0..local_dirs.len());@@ -719,6 +730,37 @@ mod tests {.unwrap();}+ #[test]+ fn test_directories_mode_creates_dirs_lazily() {+ let dir = TempDir::new().unwrap();+ let root = dir.path().to_path_buf();++ let manager = Arc::new(+ DiskManagerBuilder::default()+ .with_mode(DiskManagerMode::Directories(vec![root.clone()]))+ .build()+ .unwrap(),+ );++ // Nothing has spilled, so no working directory should exist yet.+ let entries: Vec<_> = std::fs::read_dir(&root).unwrap().collect();+ assert!(+ entries.is_empty(),+ "expected no directories before the first spill, found {entries:?}"+ );+ assert!(manager.temp_dir_paths().is_empty());++ // The first temporary file creates the working directory under the+ // configured path.+ let spill_file = manager.create_tmp_file("lazy dir test").unwrap();+ let spill_path = spill_file.path().expect("spill file should have a path");+ assert_path_in_dirs(spill_path, std::iter::once(root.as_path()));++ let temp_dir_paths = manager.temp_dir_paths();+ assert_eq!(temp_dir_paths.len(), 1);+ assert!(temp_dir_paths[0].starts_with(&root));+ }+/// Asserts that `file_path` is found anywhere in any of `dir` directoriesfn assert_path_in_dirs<'a>(file_path: &'a Path,