The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
perf(vfs): share nested repo GitignoreStacks and bound synchronous watch_tree startup (#1705)
Deng Liua268b2c488ee
diff --git a/crates/ide-vfs/src/rules.rs b/crates/ide-vfs/src/rules.rsindex cad487a4c..e6c7b66ed 100644--- a/crates/ide-vfs/src/rules.rs+++ b/crates/ide-vfs/src/rules.rs@@ -81,6 +81,14 @@ fn exclude_path(root: &Path) -> PathBuf {.unwrap_or_else(|| root.join(".git/info/exclude"))}+#[derive(Default)]+struct NestedRepos {+ /// Directory -> enclosing repository root when different from `GitignoreStack::repo_root`.+ by_dir: HashMap<PathBuf, Option<PathBuf>>,+ /// Repository root -> shared `GitignoreStack` for that nested repository.+ by_root: HashMap<PathBuf, Arc<GitignoreStack>>,+}+/// Layered gitignore matching: per-directory `.gitignore` / `.ignore` from the/// path's directory up to the repository root (or the file system root when/// not in a repository), then `.git/info/exclude` and the global excludes file.@@ -89,7 +97,7 @@ pub(crate) struct GitignoreStack {/// `.git/info/exclude` and the global excludes file, rooted at the repo root.repo_level: RwLock<Vec<Gitignore>>,cache: Mutex<HashMap<PathBuf, Option<Arc<Gitignore>>>>,- nested: Mutex<HashMap<PathBuf, Option<Arc<GitignoreStack>>>>,+ nested: Mutex<NestedRepos>,}/// How long [`GitignoreStack::cached`] reuses the repository lookup of a directory.@@ -191,13 +199,23 @@ impl GitignoreStack {if let (Some(root), Some(map)) = (&self.repo_root, REPO_LEVEL.lock().as_mut()) {map.retain(|dir, _| !dir.starts_with(root));}- self.nested.lock().clear();+ self.nested.lock().by_root.values().for_each(|s| s.reload_repo_level());*self.repo_level.write() = Self::load_repo_level(self.repo_root.as_deref());}/// Forgets the cached rules of `dir` and every directory below it.pub fn invalidate_tree(&self, dir: &Path) {- self.nested.lock().clear();+ {+ let mut nested = self.nested.lock();+ let target = if dir.file_name().is_some_and(|n| n == ".git") { dir.parent().unwrap_or(dir) } else { dir };+ if nested.by_dir.contains_key(target) || nested.by_root.contains_key(target) {+ nested.by_dir.retain(|k, _| !k.starts_with(target));+ nested.by_root.retain(|k, _| !k.starts_with(target));+ }+ for stack in nested.by_root.values() {+ stack.invalidate_tree(dir);+ }+ }let mut cache = self.cache.lock();// Lookups always populate every ancestor, so no key below `dir` exists without `dir`.if cache.contains_key(dir) {@@ -211,10 +229,57 @@ impl GitignoreStack {/// Forgets the cached rules of `dir` (after its `.gitignore` changed).pub fn invalidate(&self, dir: &Path) {- self.nested.lock().clear();+ for stack in self.nested.lock().by_root.values() {+ stack.invalidate(dir);+ }self.cache.lock().remove(dir);}+ /// Returns the shared `GitignoreStack` of the nested repository enclosing `dir`, if any.+ fn nested_for(&self, dir: &Path) -> Option<Arc<GitignoreStack>> {+ let mut nested = self.nested.lock();+ let root = if let Some(cached) = nested.by_dir.get(dir) {+ cached.clone()+ } else {+ let mut chain = Vec::new();+ let mut found = None;+ for anc in dir.ancestors() {+ if let Some(cached) = nested.by_dir.get(anc) {+ found = cached.clone();+ break;+ }+ chain.push(anc.to_path_buf());+ if self.repo_root.as_deref() == Some(anc) {+ found = None;+ break;+ }+ if anc.join(".git").exists() {+ found = Some(anc.to_path_buf());+ break;+ }+ }+ for p in chain {+ nested.by_dir.insert(p, found.clone());+ }+ found+ }?;+ Some(+ nested+ .by_root+ .entry(root.clone())+ .or_insert_with(|| {+ let repo_level = RwLock::new(Self::load_repo_level(Some(&root)));+ Arc::new(Self {+ repo_root: Some(root),+ repo_level,+ cache: Mutex::default(),+ nested: Mutex::default(),+ })+ })+ .clone(),+ )+ }+fn matcher_for(&self, dir: &Path) -> Option<Arc<Gitignore>> {if let Some(cached) = self.cache.lock().get(dir) {return cached.clone();@@ -250,20 +315,10 @@ impl GitignoreStack {pub fn is_ignored(&self, path: &Path, is_dir: bool) -> bool {// A project's watcher can span independent nested repositories. Their local// excludes and ignore ancestry must stop at their own repository boundary.- if let Some(parent) = path.parent() {- let nested = self- .nested- .lock()- .entry(parent.to_path_buf())- .or_insert_with(|| {- find_repo_root(parent)- .filter(|root| self.repo_root.as_ref() != Some(root))- .map(|_| Arc::new(Self::new(parent)))- })- .clone();- if let Some(nested) = nested {- return nested.is_ignored(path, is_dir);- }+ if let Some(parent) = path.parent()+ && let Some(nested) = self.nested_for(parent)+ {+ return nested.is_ignored(path, is_dir);}if self.repo_root.as_deref() == Some(path) {return false;@@ -302,20 +357,10 @@ impl GitignoreStack {/// ignored) still emit its own create/remove so the tree can show it/// appearing marked as ignored, while files inside it stay filtered out.pub fn is_ignored_self(&self, path: &Path, is_dir: bool) -> bool {- if let Some(parent) = path.parent() {- let nested = self- .nested- .lock()- .entry(parent.to_path_buf())- .or_insert_with(|| {- find_repo_root(parent)- .filter(|root| self.repo_root.as_ref() != Some(root))- .map(|_| Arc::new(Self::new(parent)))- })- .clone();- if let Some(nested) = nested {- return nested.is_ignored_self(path, is_dir);- }+ if let Some(parent) = path.parent()+ && let Some(nested) = self.nested_for(parent)+ {+ return nested.is_ignored_self(path, is_dir);}if self.repo_root.as_deref() == Some(path) {return false;@@ -405,4 +450,49 @@ mod tests {assert_eq!(first.repo_root(), again.repo_root());assert_eq!(again.repo_root(), Some(tmp.path()));}++ #[test]+ fn nested_repository_shares_one_gitignore_stack_across_subdirectories() {+ let tmp = tempfile::tempdir().unwrap();+ let workspace = tmp.path();+ let repo = workspace.join("yolo");+ std::fs::create_dir_all(repo.join(".git")).unwrap();+ std::fs::create_dir_all(repo.join("a/b/c")).unwrap();+ std::fs::create_dir_all(repo.join("x/y/z")).unwrap();+ std::fs::write(repo.join(".gitignore"), "*.ignored\n").unwrap();++ let stack = GitignoreStack::new(workspace);+ assert!(stack.is_ignored(&repo.join("a/b/c/file.ignored"), false));+ assert!(stack.is_ignored(&repo.join("x/y/z/other.ignored"), false));+ assert!(!stack.is_ignored(&repo.join("x/y/z/keep.rs"), false));++ assert_eq!(stack.nested.lock().by_root.len(), 1);+ let nested = stack.nested.lock().by_root.get(&repo).unwrap().clone();+ assert!(nested.cache.lock().contains_key(&repo));++ // Invalidating a leaf file or directory inside the nested repo keeps its shared stack.+ stack.invalidate_tree(&repo.join("a/b/c/new.rs"));+ assert_eq!(stack.nested.lock().by_root.len(), 1);+ assert!(Arc::ptr_eq(&nested, stack.nested.lock().by_root.get(&repo).unwrap()));+ }++ #[test]+ fn a_repository_created_inside_the_workspace_becomes_a_boundary() {+ let tmp = tempfile::tempdir().unwrap();+ let workspace = tmp.path();+ std::fs::create_dir_all(workspace.join(".git")).unwrap();+ std::fs::create_dir_all(workspace.join("sub/deep")).unwrap();+ std::fs::write(workspace.join(".gitignore"), "*.tmp\n").unwrap();++ let stack = GitignoreStack::new(workspace);+ assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));++ std::fs::create_dir(workspace.join("sub/.git")).unwrap();+ stack.invalidate_tree(&workspace.join("sub/.git"));+ assert!(!stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));++ std::fs::remove_dir(workspace.join("sub/.git")).unwrap();+ stack.invalidate_tree(&workspace.join("sub/.git"));+ assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));+ }}diff --git a/crates/ide-vfs/src/watch.rs b/crates/ide-vfs/src/watch.rsindex 253d7a826..47babf368 100644--- a/crates/ide-vfs/src/watch.rs+++ b/crates/ide-vfs/src/watch.rs@@ -27,6 +27,9 @@ const DEBOUNCE: Duration = Duration::from_millis(100);const MAX_DELAY: Duration = Duration::from_millis(1000);/// How often the polling fallback rescans the watched directories.const POLL_INTERVAL: Duration = Duration::from_secs(2);+/// Directories watched synchronously in [`Watcher::start`] before handing the+/// remainder of a large tree to the watcher thread.+const SYNC_WATCH_BUDGET: usize = 256;/// How directories are registered with the OS.#[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -178,11 +181,12 @@ impl Watcher {).map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;*state.watcher.lock() = Some(Box::new(watcher));- if mode == WatchMode::PerDirectory {- state.watch_tree(root, false, &mut Vec::new());+ let deferred = if mode == WatchMode::PerDirectory {+ state.watch_tree_bounded(root, SYNC_WATCH_BUDGET)} else {state.add_watch(root);- }+ Vec::new()+ };if state.watched.lock().is_empty() {return Err(io_error(format!("cannot watch {}", root.display())));}@@ -190,7 +194,7 @@ impl Watcher {let state = state.clone();std::thread::Builder::new().name("openidea-fs-watch".into())- .spawn(move || state.run(&*emit))+ .spawn(move || state.run(deferred, &*emit)).map_err(|e| io_error(format!("cannot start file watcher: {e}")))?};Ok(Inner { state, thread: Some(thread) })@@ -239,8 +243,12 @@ impl DirState {self.stop.load(Ordering::SeqCst)}- /// Debounce loop: waits for a quiet period, then processes the batch.- fn run(&self, emit: impl Fn(&FsBatch)) {+ /// Debounce loop: finishes any deferred startup watches, then waits for a+ /// quiet period and processes each batch.+ fn run(&self, deferred: Vec<(PathBuf, bool)>, emit: impl Fn(&FsBatch)) {+ if !deferred.is_empty() {+ self.watch_stack(deferred, false, &mut Vec::new(), None);+ }loop {let events = {let mut q = self.queue.lock();@@ -298,6 +306,7 @@ impl DirState {// during this batch would otherwise never be seen.for path in &batch.vcs {if !self.watched.lock().contains(path) && path.symlink_metadata().is_ok_and(|m| m.is_dir()) {+ self.filter.gitignore.invalidate_tree(path);self.watch_tree(path, false, &mut discovered);}}@@ -349,11 +358,33 @@ impl DirState {/// files and directories found inside (created before the watch was in/// place) are appended to `found`.fn watch_tree(&self, dir: &Path, report: bool, found: &mut Vec<PathBuf>) {- let mut stack = vec![dir.to_path_buf()];- while let Some(dir) = stack.pop() {- if self.stopped() || !self.filter.should_watch_dir(&dir) || !self.add_watch(&dir) {+ self.watch_stack(vec![(dir.to_path_buf(), false)], report, found, None);+ }++ /// Watches up to `budget` directories synchronously and returns any+ /// remaining `(dir, parent_verified)` entries for background registration.+ fn watch_tree_bounded(&self, dir: &Path, budget: usize) -> Vec<(PathBuf, bool)> {+ self.watch_stack(vec![(dir.to_path_buf(), false)], false, &mut Vec::new(), Some(budget))+ }++ fn watch_stack(+ &self,+ mut stack: Vec<(PathBuf, bool)>,+ report: bool,+ found: &mut Vec<PathBuf>,+ budget: Option<usize>,+ ) -> Vec<(PathBuf, bool)> {+ let mut watched_now = 0;+ while let Some((dir, parent_ok)) = stack.pop() {+ if self.stopped() {+ break;+ }+ let should_watch =+ if parent_ok { self.filter.should_watch_child_dir(&dir) } else { self.filter.should_watch_dir(&dir) };+ if !should_watch || !self.add_watch(&dir) {continue;}+ watched_now += 1;let Ok(entries) = std::fs::read_dir(&dir) else { continue };for entry in entries.flatten() {let path = entry.path();@@ -362,10 +393,14 @@ impl DirState {found.push(path.clone());}if is_dir {- stack.push(path);+ stack.push((path, true));}}+ if budget.is_some_and(|max| watched_now >= max) {+ break;+ }}+ stack}fn unwatch_tree(&self, dir: &Path) {@@ -469,6 +504,15 @@ impl EventFilter {/// and `worktrees/` (HEAD, index, refs and worktree metadata are what VCS/// refresh needs).pub fn should_watch_dir(&self, dir: &Path) -> bool {+ self.should_watch_dir_inner(dir, false)+ }++ /// Fast path when `dir`'s parent directory is already known to be watchable.+ fn should_watch_child_dir(&self, dir: &Path) -> bool {+ self.should_watch_dir_inner(dir, true)+ }++ fn should_watch_dir_inner(&self, dir: &Path, parent_ok: bool) -> bool {let Ok(rel) = dir.strip_prefix(&self.root) else { return false };if rel.as_os_str().is_empty() {return true;@@ -484,7 +528,11 @@ impl EventFilter {_ => false,};}- !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)+ if parent_ok {+ !self.excludes.read().is_match(rel) && !self.gitignore.is_ignored_self(dir, true)+ } else {+ !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)+ }}/// Whether a path found by scanning a new directory would be reported.@@ -1075,6 +1123,42 @@ mod tests {assert_eq!(batches.lock().len(), n, "no events after drop");}+ #[test]+ fn per_directory_mode_watches_the_rest_of_a_large_tree_after_start() {+ let tmp = tempfile::tempdir().unwrap();+ let root = dunce::canonicalize(tmp.path()).unwrap();+ let dirs = SYNC_WATCH_BUDGET + 40;+ for i in 0..dirs {+ std::fs::create_dir_all(root.join(format!("d{i:03}"))).unwrap();+ }+ std::fs::create_dir_all(root.join("node_modules/x")).unwrap();+ let batches: Arc<Mutex<Vec<FsBatch>>> = Arc::default();+ let sink: SharedSink = Arc::new(|_e: IdeEvent| {});+ let watcher = {+ let batches = batches.clone();+ Watcher::start_with(+ &root,+ vec!["**/node_modules".into()],+ sink,+ Box::new(move |b| batches.lock().push(b.clone())),+ WatchMode::PerDirectory,+ )+ .unwrap()+ };+ let deadline = Instant::now() + Duration::from_secs(10);+ while watcher.watch_count() < dirs + 1 && Instant::now() < deadline {+ std::thread::sleep(Duration::from_millis(20));+ }+ // root and every d*, but not node_modules+ assert_eq!(watcher.watch_count(), dirs + 1);+ std::thread::sleep(Duration::from_millis(200));+ for i in [0, dirs - 1] {+ let file = root.join(format!("d{i:03}/f.rs"));+ std::fs::write(&file, "x").unwrap();+ assert!(wait_for(&batches, |b| all_paths(b).contains(&file)), "{file:?}");+ }+ }+#[test]fn per_directory_mode_follows_worktree_registrations() {let tmp = tempfile::tempdir().unwrap();