Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

perf(vfs): share nested repo GitignoreStacks and bound synchronous watch_tree startup (#1705)

Deng Liua268b2c488ee
diff --git a/crates/ide-vfs/src/rules.rs b/crates/ide-vfs/src/rules.rs
index cad487a4c..e6c7b66ed 100644
--- a/crates/ide-vfs/src/rules.rs
+++ b/crates/ide-vfs/src/rules.rs
@@ -81,6 +81,14 @@ fn exclude_path(root: &Path) -> PathBuf {
.unwrap_or_else(|| root.join(".git/info/exclude"))
}
+#[derive(Default)]
+struct NestedRepos {
+ /// Directory -> enclosing repository root when different from `GitignoreStack::repo_root`.
+ by_dir: HashMap<PathBuf, Option<PathBuf>>,
+ /// Repository root -> shared `GitignoreStack` for that nested repository.
+ by_root: HashMap<PathBuf, Arc<GitignoreStack>>,
+}
+
/// Layered gitignore matching: per-directory `.gitignore` / `.ignore` from the
/// path's directory up to the repository root (or the file system root when
/// not in a repository), then `.git/info/exclude` and the global excludes file.
@@ -89,7 +97,7 @@ pub(crate) struct GitignoreStack {
/// `.git/info/exclude` and the global excludes file, rooted at the repo root.
repo_level: RwLock<Vec<Gitignore>>,
cache: Mutex<HashMap<PathBuf, Option<Arc<Gitignore>>>>,
- nested: Mutex<HashMap<PathBuf, Option<Arc<GitignoreStack>>>>,
+ nested: Mutex<NestedRepos>,
}
/// How long [`GitignoreStack::cached`] reuses the repository lookup of a directory.
@@ -191,13 +199,23 @@ impl GitignoreStack {
if let (Some(root), Some(map)) = (&self.repo_root, REPO_LEVEL.lock().as_mut()) {
map.retain(|dir, _| !dir.starts_with(root));
}
- self.nested.lock().clear();
+ self.nested.lock().by_root.values().for_each(|s| s.reload_repo_level());
*self.repo_level.write() = Self::load_repo_level(self.repo_root.as_deref());
}
/// Forgets the cached rules of `dir` and every directory below it.
pub fn invalidate_tree(&self, dir: &Path) {
- self.nested.lock().clear();
+ {
+ let mut nested = self.nested.lock();
+ let target = if dir.file_name().is_some_and(|n| n == ".git") { dir.parent().unwrap_or(dir) } else { dir };
+ if nested.by_dir.contains_key(target) || nested.by_root.contains_key(target) {
+ nested.by_dir.retain(|k, _| !k.starts_with(target));
+ nested.by_root.retain(|k, _| !k.starts_with(target));
+ }
+ for stack in nested.by_root.values() {
+ stack.invalidate_tree(dir);
+ }
+ }
let mut cache = self.cache.lock();
// Lookups always populate every ancestor, so no key below `dir` exists without `dir`.
if cache.contains_key(dir) {
@@ -211,10 +229,57 @@ impl GitignoreStack {
/// Forgets the cached rules of `dir` (after its `.gitignore` changed).
pub fn invalidate(&self, dir: &Path) {
- self.nested.lock().clear();
+ for stack in self.nested.lock().by_root.values() {
+ stack.invalidate(dir);
+ }
self.cache.lock().remove(dir);
}
+ /// Returns the shared `GitignoreStack` of the nested repository enclosing `dir`, if any.
+ fn nested_for(&self, dir: &Path) -> Option<Arc<GitignoreStack>> {
+ let mut nested = self.nested.lock();
+ let root = if let Some(cached) = nested.by_dir.get(dir) {
+ cached.clone()
+ } else {
+ let mut chain = Vec::new();
+ let mut found = None;
+ for anc in dir.ancestors() {
+ if let Some(cached) = nested.by_dir.get(anc) {
+ found = cached.clone();
+ break;
+ }
+ chain.push(anc.to_path_buf());
+ if self.repo_root.as_deref() == Some(anc) {
+ found = None;
+ break;
+ }
+ if anc.join(".git").exists() {
+ found = Some(anc.to_path_buf());
+ break;
+ }
+ }
+ for p in chain {
+ nested.by_dir.insert(p, found.clone());
+ }
+ found
+ }?;
+ Some(
+ nested
+ .by_root
+ .entry(root.clone())
+ .or_insert_with(|| {
+ let repo_level = RwLock::new(Self::load_repo_level(Some(&root)));
+ Arc::new(Self {
+ repo_root: Some(root),
+ repo_level,
+ cache: Mutex::default(),
+ nested: Mutex::default(),
+ })
+ })
+ .clone(),
+ )
+ }
+
fn matcher_for(&self, dir: &Path) -> Option<Arc<Gitignore>> {
if let Some(cached) = self.cache.lock().get(dir) {
return cached.clone();
@@ -250,20 +315,10 @@ impl GitignoreStack {
pub fn is_ignored(&self, path: &Path, is_dir: bool) -> bool {
// A project's watcher can span independent nested repositories. Their local
// excludes and ignore ancestry must stop at their own repository boundary.
- if let Some(parent) = path.parent() {
- let nested = self
- .nested
- .lock()
- .entry(parent.to_path_buf())
- .or_insert_with(|| {
- find_repo_root(parent)
- .filter(|root| self.repo_root.as_ref() != Some(root))
- .map(|_| Arc::new(Self::new(parent)))
- })
- .clone();
- if let Some(nested) = nested {
- return nested.is_ignored(path, is_dir);
- }
+ if let Some(parent) = path.parent()
+ && let Some(nested) = self.nested_for(parent)
+ {
+ return nested.is_ignored(path, is_dir);
}
if self.repo_root.as_deref() == Some(path) {
return false;
@@ -302,20 +357,10 @@ impl GitignoreStack {
/// ignored) still emit its own create/remove so the tree can show it
/// appearing marked as ignored, while files inside it stay filtered out.
pub fn is_ignored_self(&self, path: &Path, is_dir: bool) -> bool {
- if let Some(parent) = path.parent() {
- let nested = self
- .nested
- .lock()
- .entry(parent.to_path_buf())
- .or_insert_with(|| {
- find_repo_root(parent)
- .filter(|root| self.repo_root.as_ref() != Some(root))
- .map(|_| Arc::new(Self::new(parent)))
- })
- .clone();
- if let Some(nested) = nested {
- return nested.is_ignored_self(path, is_dir);
- }
+ if let Some(parent) = path.parent()
+ && let Some(nested) = self.nested_for(parent)
+ {
+ return nested.is_ignored_self(path, is_dir);
}
if self.repo_root.as_deref() == Some(path) {
return false;
@@ -405,4 +450,49 @@ mod tests {
assert_eq!(first.repo_root(), again.repo_root());
assert_eq!(again.repo_root(), Some(tmp.path()));
}
+
+ #[test]
+ fn nested_repository_shares_one_gitignore_stack_across_subdirectories() {
+ let tmp = tempfile::tempdir().unwrap();
+ let workspace = tmp.path();
+ let repo = workspace.join("yolo");
+ std::fs::create_dir_all(repo.join(".git")).unwrap();
+ std::fs::create_dir_all(repo.join("a/b/c")).unwrap();
+ std::fs::create_dir_all(repo.join("x/y/z")).unwrap();
+ std::fs::write(repo.join(".gitignore"), "*.ignored\n").unwrap();
+
+ let stack = GitignoreStack::new(workspace);
+ assert!(stack.is_ignored(&repo.join("a/b/c/file.ignored"), false));
+ assert!(stack.is_ignored(&repo.join("x/y/z/other.ignored"), false));
+ assert!(!stack.is_ignored(&repo.join("x/y/z/keep.rs"), false));
+
+ assert_eq!(stack.nested.lock().by_root.len(), 1);
+ let nested = stack.nested.lock().by_root.get(&repo).unwrap().clone();
+ assert!(nested.cache.lock().contains_key(&repo));
+
+ // Invalidating a leaf file or directory inside the nested repo keeps its shared stack.
+ stack.invalidate_tree(&repo.join("a/b/c/new.rs"));
+ assert_eq!(stack.nested.lock().by_root.len(), 1);
+ assert!(Arc::ptr_eq(&nested, stack.nested.lock().by_root.get(&repo).unwrap()));
+ }
+
+ #[test]
+ fn a_repository_created_inside_the_workspace_becomes_a_boundary() {
+ let tmp = tempfile::tempdir().unwrap();
+ let workspace = tmp.path();
+ std::fs::create_dir_all(workspace.join(".git")).unwrap();
+ std::fs::create_dir_all(workspace.join("sub/deep")).unwrap();
+ std::fs::write(workspace.join(".gitignore"), "*.tmp\n").unwrap();
+
+ let stack = GitignoreStack::new(workspace);
+ assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+
+ std::fs::create_dir(workspace.join("sub/.git")).unwrap();
+ stack.invalidate_tree(&workspace.join("sub/.git"));
+ assert!(!stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+
+ std::fs::remove_dir(workspace.join("sub/.git")).unwrap();
+ stack.invalidate_tree(&workspace.join("sub/.git"));
+ assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+ }
}
diff --git a/crates/ide-vfs/src/watch.rs b/crates/ide-vfs/src/watch.rs
index 253d7a826..47babf368 100644
--- a/crates/ide-vfs/src/watch.rs
+++ b/crates/ide-vfs/src/watch.rs
@@ -27,6 +27,9 @@ const DEBOUNCE: Duration = Duration::from_millis(100);
const MAX_DELAY: Duration = Duration::from_millis(1000);
/// How often the polling fallback rescans the watched directories.
const POLL_INTERVAL: Duration = Duration::from_secs(2);
+/// Directories watched synchronously in [`Watcher::start`] before handing the
+/// remainder of a large tree to the watcher thread.
+const SYNC_WATCH_BUDGET: usize = 256;
/// How directories are registered with the OS.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -178,11 +181,12 @@ impl Watcher {
)
.map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;
*state.watcher.lock() = Some(Box::new(watcher));
- if mode == WatchMode::PerDirectory {
- state.watch_tree(root, false, &mut Vec::new());
+ let deferred = if mode == WatchMode::PerDirectory {
+ state.watch_tree_bounded(root, SYNC_WATCH_BUDGET)
} else {
state.add_watch(root);
- }
+ Vec::new()
+ };
if state.watched.lock().is_empty() {
return Err(io_error(format!("cannot watch {}", root.display())));
}
@@ -190,7 +194,7 @@ impl Watcher {
let state = state.clone();
std::thread::Builder::new()
.name("openidea-fs-watch".into())
- .spawn(move || state.run(&*emit))
+ .spawn(move || state.run(deferred, &*emit))
.map_err(|e| io_error(format!("cannot start file watcher: {e}")))?
};
Ok(Inner { state, thread: Some(thread) })
@@ -239,8 +243,12 @@ impl DirState {
self.stop.load(Ordering::SeqCst)
}
- /// Debounce loop: waits for a quiet period, then processes the batch.
- fn run(&self, emit: impl Fn(&FsBatch)) {
+ /// Debounce loop: finishes any deferred startup watches, then waits for a
+ /// quiet period and processes each batch.
+ fn run(&self, deferred: Vec<(PathBuf, bool)>, emit: impl Fn(&FsBatch)) {
+ if !deferred.is_empty() {
+ self.watch_stack(deferred, false, &mut Vec::new(), None);
+ }
loop {
let events = {
let mut q = self.queue.lock();
@@ -298,6 +306,7 @@ impl DirState {
// during this batch would otherwise never be seen.
for path in &batch.vcs {
if !self.watched.lock().contains(path) && path.symlink_metadata().is_ok_and(|m| m.is_dir()) {
+ self.filter.gitignore.invalidate_tree(path);
self.watch_tree(path, false, &mut discovered);
}
}
@@ -349,11 +358,33 @@ impl DirState {
/// files and directories found inside (created before the watch was in
/// place) are appended to `found`.
fn watch_tree(&self, dir: &Path, report: bool, found: &mut Vec<PathBuf>) {
- let mut stack = vec![dir.to_path_buf()];
- while let Some(dir) = stack.pop() {
- if self.stopped() || !self.filter.should_watch_dir(&dir) || !self.add_watch(&dir) {
+ self.watch_stack(vec![(dir.to_path_buf(), false)], report, found, None);
+ }
+
+ /// Watches up to `budget` directories synchronously and returns any
+ /// remaining `(dir, parent_verified)` entries for background registration.
+ fn watch_tree_bounded(&self, dir: &Path, budget: usize) -> Vec<(PathBuf, bool)> {
+ self.watch_stack(vec![(dir.to_path_buf(), false)], false, &mut Vec::new(), Some(budget))
+ }
+
+ fn watch_stack(
+ &self,
+ mut stack: Vec<(PathBuf, bool)>,
+ report: bool,
+ found: &mut Vec<PathBuf>,
+ budget: Option<usize>,
+ ) -> Vec<(PathBuf, bool)> {
+ let mut watched_now = 0;
+ while let Some((dir, parent_ok)) = stack.pop() {
+ if self.stopped() {
+ break;
+ }
+ let should_watch =
+ if parent_ok { self.filter.should_watch_child_dir(&dir) } else { self.filter.should_watch_dir(&dir) };
+ if !should_watch || !self.add_watch(&dir) {
continue;
}
+ watched_now += 1;
let Ok(entries) = std::fs::read_dir(&dir) else { continue };
for entry in entries.flatten() {
let path = entry.path();
@@ -362,10 +393,14 @@ impl DirState {
found.push(path.clone());
}
if is_dir {
- stack.push(path);
+ stack.push((path, true));
}
}
+ if budget.is_some_and(|max| watched_now >= max) {
+ break;
+ }
}
+ stack
}
fn unwatch_tree(&self, dir: &Path) {
@@ -469,6 +504,15 @@ impl EventFilter {
/// and `worktrees/` (HEAD, index, refs and worktree metadata are what VCS
/// refresh needs).
pub fn should_watch_dir(&self, dir: &Path) -> bool {
+ self.should_watch_dir_inner(dir, false)
+ }
+
+ /// Fast path when `dir`'s parent directory is already known to be watchable.
+ fn should_watch_child_dir(&self, dir: &Path) -> bool {
+ self.should_watch_dir_inner(dir, true)
+ }
+
+ fn should_watch_dir_inner(&self, dir: &Path, parent_ok: bool) -> bool {
let Ok(rel) = dir.strip_prefix(&self.root) else { return false };
if rel.as_os_str().is_empty() {
return true;
@@ -484,7 +528,11 @@ impl EventFilter {
_ => false,
};
}
- !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)
+ if parent_ok {
+ !self.excludes.read().is_match(rel) && !self.gitignore.is_ignored_self(dir, true)
+ } else {
+ !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)
+ }
}
/// Whether a path found by scanning a new directory would be reported.
@@ -1075,6 +1123,42 @@ mod tests {
assert_eq!(batches.lock().len(), n, "no events after drop");
}
+ #[test]
+ fn per_directory_mode_watches_the_rest_of_a_large_tree_after_start() {
+ let tmp = tempfile::tempdir().unwrap();
+ let root = dunce::canonicalize(tmp.path()).unwrap();
+ let dirs = SYNC_WATCH_BUDGET + 40;
+ for i in 0..dirs {
+ std::fs::create_dir_all(root.join(format!("d{i:03}"))).unwrap();
+ }
+ std::fs::create_dir_all(root.join("node_modules/x")).unwrap();
+ let batches: Arc<Mutex<Vec<FsBatch>>> = Arc::default();
+ let sink: SharedSink = Arc::new(|_e: IdeEvent| {});
+ let watcher = {
+ let batches = batches.clone();
+ Watcher::start_with(
+ &root,
+ vec!["**/node_modules".into()],
+ sink,
+ Box::new(move |b| batches.lock().push(b.clone())),
+ WatchMode::PerDirectory,
+ )
+ .unwrap()
+ };
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while watcher.watch_count() < dirs + 1 && Instant::now() < deadline {
+ std::thread::sleep(Duration::from_millis(20));
+ }
+ // root and every d*, but not node_modules
+ assert_eq!(watcher.watch_count(), dirs + 1);
+ std::thread::sleep(Duration::from_millis(200));
+ for i in [0, dirs - 1] {
+ let file = root.join(format!("d{i:03}/f.rs"));
+ std::fs::write(&file, "x").unwrap();
+ assert!(wait_for(&batches, |b| all_paths(b).contains(&file)), "{file:?}");
+ }
+ }
+
#[test]
fn per_directory_mode_follows_worktree_registrations() {
let tmp = tempfile::tempdir().unwrap();