Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

fix(vfs): poll for changes when the OS file watcher cannot start (#906)

Azeem Hassniecdd43a55ae6
diff --git a/crates/ide-host/src/project.rs b/crates/ide-host/src/project.rs
index 1391f6182..e7d26808c 100644
--- a/crates/ide-host/src/project.rs
+++ b/crates/ide-host/src/project.rs
@@ -365,6 +365,13 @@ impl Host {
}),
)?
};
+ if let Some(reason) = watcher.polling() {
+ sink.emit(IdeEvent::Notify {
+ level: ide_core::event::NotifyLevel::Warning,
+ title: "External file changes sync may be slow".into(),
+ body: Some(reason.into()),
+ });
+ }
// Scratch files live outside the project; watch them too so the
// Project view and open editors follow changes made there.
diff --git a/crates/ide-vfs/src/watch.rs b/crates/ide-vfs/src/watch.rs
index 011cab690..d9309d90b 100644
--- a/crates/ide-vfs/src/watch.rs
+++ b/crates/ide-vfs/src/watch.rs
@@ -5,12 +5,15 @@
//! that skips excluded / git-ignored directories (Linux, where a recursive
//! inotify watch would register every directory of `node_modules` and
//! `target` and exhaust `max_user_watches`).
+//!
+//! When the OS watcher cannot start (inotify limits used up, a file system
+//! without change notifications), the same directories are polled instead.
use crate::FsBatch;
use crate::rules::{ExcludeMatcher, GitignoreStack};
use ide_core::{IdeError, IdeEvent, IdeResult, SharedSink};
use notify_debouncer_full::notify::event::{ModifyKind, RenameMode};
-use notify_debouncer_full::notify::{self, EventKind, RecommendedWatcher, RecursiveMode, Watcher as _};
+use notify_debouncer_full::notify::{self, EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, WatcherKind};
use notify_debouncer_full::{DebounceEventResult, DebouncedEvent, Debouncer, NoCache, new_debouncer_opt};
use parking_lot::{Condvar, Mutex, RwLock};
use std::collections::{BTreeMap, BTreeSet, HashSet};
@@ -23,6 +26,8 @@ use std::time::{Duration, Instant};
const DEBOUNCE: Duration = Duration::from_millis(100);
/// Upper bound on how long a continuous stream of events is held back.
const MAX_DELAY: Duration = Duration::from_millis(1000);
+/// How often the polling fallback rescans the watched directories.
+const POLL_INTERVAL: Duration = Duration::from_secs(2);
/// How directories are registered with the OS.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -43,6 +48,7 @@ impl Default for WatchMode {
}
type OnChange = Box<dyn Fn(&FsBatch) + Send + Sync>;
+type Emit = Arc<dyn Fn(&FsBatch) + Send + Sync>;
/// Debounced watcher. Emits `IdeEvent::FsChanged` to `sink` and invokes
/// `on_change` (used by the index) with the same batch. Stops when dropped;
@@ -51,12 +57,14 @@ pub struct Watcher {
inner: Inner,
excludes: SharedExcludes,
rules: Arc<GitignoreStack>,
+ polling: Option<String>,
}
type SharedExcludes = Arc<RwLock<ExcludeMatcher>>;
enum Inner {
Recursive(#[allow(dead_code)] Debouncer<RecommendedWatcher, NoCache>),
+ Polling(#[allow(dead_code)] Debouncer<PollWatcher, NoCache>),
PerDirectory { state: Arc<DirState>, thread: Option<JoinHandle<()>> },
}
@@ -76,7 +84,7 @@ impl Watcher {
let filter = EventFilter::new(root.clone(), &excludes);
let shared = filter.excludes.clone();
let rules = filter.gitignore.clone();
- let emit = move |batch: &FsBatch| {
+ let emit: Emit = Arc::new(move |batch: &FsBatch| {
// VCS-metadata-only batches (`batch.vcs`) carry no worktree paths:
// consumers that refresh on any `fsChanged` would reschedule purely
// from the backend's own `.git/index` write.
@@ -91,13 +99,21 @@ impl Watcher {
sink.emit(IdeEvent::ExcludedDirsChanged { dirs: batch.excluded_dirs.iter().map(Into::into).collect() });
}
on_change(batch);
+ });
+ let (inner, polling) = match Self::start_native(&root, filter.clone(), emit.clone(), mode) {
+ Ok(inner) => (inner, None),
+ // Rather than failing to open the project, keep it in sync a little later.
+ Err(e) => {
+ tracing::warn!("{e}; polling {} for changes instead", root.display());
+ (Self::start_polling(&root, filter, emit, mode)?, Some(e.message))
+ }
};
- let inner = match mode {
- WatchMode::Recursive => Self::start_recursive(root, filter, emit, RecursiveMode::Recursive),
- WatchMode::Flat => Self::start_recursive(root, filter, emit, RecursiveMode::NonRecursive),
- WatchMode::PerDirectory => Self::start_per_directory(root, filter, emit),
- }?;
- Ok(Self { inner, excludes: shared, rules })
+ Ok(Self { inner, excludes: shared, rules, polling })
+ }
+
+ /// Why changes are polled for instead of reported by the OS; `None` when they are.
+ pub fn polling(&self) -> Option<&str> {
+ self.polling.as_deref()
}
/// Replaces the exclude globs (Settings › Project Structure / File Types
@@ -116,21 +132,53 @@ impl Watcher {
/// Number of OS-level watches (1 in recursive and flat mode).
pub fn watch_count(&self) -> usize {
match &self.inner {
- Inner::Recursive(_) => 1,
+ Inner::Recursive(_) | Inner::Polling(_) => 1,
Inner::PerDirectory { state, .. } => state.watched.lock().len(),
}
}
- fn start_recursive(
- root: PathBuf,
+ fn start_native(root: &Path, filter: EventFilter, emit: Emit, mode: WatchMode) -> IdeResult<Inner> {
+ let config = notify::Config::default();
+ match mode {
+ WatchMode::Recursive => {
+ Self::start_debounced(root, filter, emit, RecursiveMode::Recursive, config).map(Inner::Recursive)
+ }
+ WatchMode::Flat => {
+ Self::start_debounced(root, filter, emit, RecursiveMode::NonRecursive, config).map(Inner::Recursive)
+ }
+ WatchMode::PerDirectory => Self::start_per_directory::<RecommendedWatcher>(root, filter, emit, config),
+ }
+ }
+
+ /// Polls per directory unless flat: a recursive poll would rescan
+ /// `node_modules` and `target` every round.
+ fn start_polling(root: &Path, filter: EventFilter, emit: Emit, mode: WatchMode) -> IdeResult<Inner> {
+ let config = notify::Config::default().with_poll_interval(POLL_INTERVAL);
+ match mode {
+ WatchMode::Flat => {
+ Self::start_debounced(root, filter, emit, RecursiveMode::NonRecursive, config).map(Inner::Polling)
+ }
+ WatchMode::Recursive | WatchMode::PerDirectory => {
+ Self::start_per_directory::<PollWatcher>(root, filter, emit, config)
+ }
+ }
+ }
+
+ fn start_debounced<W: notify::Watcher>(
+ root: &Path,
filter: EventFilter,
- emit: impl Fn(&FsBatch) + Send + 'static,
+ emit: Emit,
recursive: RecursiveMode,
- ) -> IdeResult<Inner> {
+ config: notify::Config,
+ ) -> IdeResult<Debouncer<W, NoCache>> {
+ let polled = W::kind() == WatcherKind::PollWatcher;
// No file-id cache: building it walks and stats the whole tree before `watch` returns,
// which made opening large projects block for seconds. Renames arrive as remove + create.
let handler = move |result: DebounceEventResult| match result {
- Ok(events) => {
+ Ok(mut events) => {
+ if polled {
+ events.retain(|e| !is_dir_touch(&e.event));
+ }
let (batch, _) = filter.batch_detailed(&events);
if !batch.is_empty() {
emit(&batch);
@@ -142,18 +190,19 @@ impl Watcher {
}
}
};
- let mut debouncer =
- new_debouncer_opt::<_, RecommendedWatcher, _>(DEBOUNCE, None, handler, NoCache, notify::Config::default())
- .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;
- debouncer.watch(&root, recursive).map_err(|e| io_error(format!("cannot watch {}: {e}", root.display())))?;
- Ok(Inner::Recursive(debouncer))
+ let mut debouncer = new_debouncer_opt::<_, W, _>(DEBOUNCE, None, handler, NoCache, config)
+ .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;
+ debouncer.watch(root, recursive).map_err(|e| io_error(format!("cannot watch {}: {e}", root.display())))?;
+ Ok(debouncer)
}
- fn start_per_directory(
- root: PathBuf,
+ fn start_per_directory<W: notify::Watcher + Send + 'static>(
+ root: &Path,
filter: EventFilter,
- emit: impl Fn(&FsBatch) + Send + 'static,
+ emit: Emit,
+ config: notify::Config,
) -> IdeResult<Inner> {
+ let polled = W::kind() == WatcherKind::PollWatcher;
let state = Arc::new(DirState {
filter,
watcher: Mutex::new(None),
@@ -163,10 +212,11 @@ impl Watcher {
stop: AtomicBool::new(false),
});
let weak: Weak<DirState> = Arc::downgrade(&state);
- let watcher = RecommendedWatcher::new(
+ let watcher = W::new(
move |event: notify::Result<notify::Event>| {
let Some(state) = weak.upgrade() else { return };
match event {
+ Ok(event) if polled && is_dir_touch(&event) => {}
Ok(event) => {
state.queue.lock().push((event, Instant::now()));
state.wake.notify_one();
@@ -174,12 +224,12 @@ impl Watcher {
Err(e) => tracing::warn!("file watcher: {e}"),
}
},
- notify::Config::default(),
+ config,
)
.map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;
- *state.watcher.lock() = Some(watcher);
+ *state.watcher.lock() = Some(Box::new(watcher));
let mut created = Vec::new();
- state.watch_tree(&root, false, &mut created);
+ state.watch_tree(root, false, &mut created);
if state.watched.lock().is_empty() {
return Err(io_error(format!("cannot watch {}", root.display())));
}
@@ -187,7 +237,7 @@ impl Watcher {
let state = state.clone();
std::thread::Builder::new()
.name("openidea-fs-watch".into())
- .spawn(move || state.run(emit))
+ .spawn(move || state.run(&*emit))
.map_err(|e| io_error(format!("cannot start file watcher: {e}")))?
};
Ok(Inner::PerDirectory { state, thread: Some(thread) })
@@ -212,11 +262,17 @@ fn io_error(msg: String) -> IdeError {
IdeError::new(ide_core::error::ErrorKind::Io, msg)
}
+/// A directory's own mtime moving as entries come and go: polling sees it, the
+/// OS watchers report only the entries (also seen, through that directory's watch).
+fn is_dir_touch(event: &notify::Event) -> bool {
+ matches!(event.kind, EventKind::Modify(_)) && event.paths.iter().all(|p| p.is_dir())
+}
+
/// State of the per-directory strategy, shared with the notify callback
/// (weakly) and the debounce thread.
struct DirState {
filter: EventFilter,
- watcher: Mutex<Option<RecommendedWatcher>>,
+ watcher: Mutex<Option<Box<dyn notify::Watcher + Send>>>,
watched: Mutex<HashSet<PathBuf>>,
queue: Mutex<Vec<(notify::Event, Instant)>>,
wake: Condvar,
@@ -396,6 +452,7 @@ fn merge(prev: Option<Change>, next: Change) -> Change {
}
}
+#[derive(Clone)]
pub(crate) struct EventFilter {
root: PathBuf,
excludes: SharedExcludes,
@@ -791,7 +848,7 @@ mod tests {
fn watcher_state(w: &Watcher) -> &DirState {
match &w.inner {
Inner::PerDirectory { state, .. } => state,
- Inner::Recursive(_) => panic!("not per-directory"),
+ Inner::Recursive(_) | Inner::Polling(_) => panic!("not per-directory"),
}
}
@@ -969,4 +1026,48 @@ mod tests {
std::thread::sleep(Duration::from_millis(300));
assert!(!all_paths(&batches.lock()).contains(&root.join("sub/deep.txt")));
}
+
+ /// The fallback used when the OS watcher cannot start.
+ fn polling_watcher(root: &Path, excludes: Vec<String>, batches: Arc<Mutex<Vec<FsBatch>>>) -> Watcher {
+ let filter = EventFilter::new(root.to_path_buf(), &excludes);
+ let (excludes, rules) = (filter.excludes.clone(), filter.gitignore.clone());
+ let emit: Emit = Arc::new(move |b: &FsBatch| batches.lock().push(b.clone()));
+ let inner = Watcher::start_polling(root, filter, emit, WatchMode::default()).unwrap();
+ Watcher { inner, excludes, rules, polling: Some("cannot start file watcher".into()) }
+ }
+
+ #[test]
+ fn polling_skips_excluded_dirs_and_follows_new_ones() {
+ let tmp = tempfile::tempdir().unwrap();
+ let root = dunce::canonicalize(tmp.path()).unwrap();
+ for d in ["src/a", "node_modules/x"] {
+ std::fs::create_dir_all(root.join(d)).unwrap();
+ }
+ let batches: Arc<Mutex<Vec<FsBatch>>> = Arc::default();
+ let watcher = polling_watcher(&root, vec!["**/node_modules".into()], batches.clone());
+ // root, src, src/a
+ assert_eq!(watcher.watch_count(), 3);
+
+ std::fs::write(root.join("node_modules/x/z.js"), "x").unwrap();
+ std::fs::write(root.join("src/a/f.rs"), "x").unwrap();
+ std::fs::create_dir_all(root.join("src/new/inner")).unwrap();
+ std::fs::write(root.join("src/new/inner/g.rs"), "x").unwrap();
+ assert!(
+ wait_for(&batches, |b| {
+ let paths = all_paths(b);
+ paths.contains(&root.join("src/a/f.rs")) && paths.contains(&root.join("src/new/inner/g.rs"))
+ }),
+ "{:?}",
+ batches.lock()
+ );
+ let paths = all_paths(&batches.lock());
+ assert!(!paths.iter().any(|p| p.starts_with(root.join("node_modules"))), "{paths:?}");
+ // Folders whose entries changed aren't reported as modified, as with inotify.
+ assert!(!paths.contains(&root.join("src")) && !paths.contains(&root.join("src/a")), "{paths:?}");
+ assert_eq!(watcher.watch_count(), 5);
+
+ // Later writes inside a new folder are seen through its own poll.
+ std::fs::write(root.join("src/new/inner/h.rs"), "x").unwrap();
+ assert!(wait_for(&batches, |b| all_paths(b).contains(&root.join("src/new/inner/h.rs"))));
+ }
}
diff --git a/src/platform/locales/text/zh.json b/src/platform/locales/text/zh.json
index 0522011de..f965fcadc 100644
--- a/src/platform/locales/text/zh.json
+++ b/src/platform/locales/text/zh.json
@@ -5547,5 +5547,6 @@
"Passed as --settings; empty keeps DJANGO_SETTINGS_MODULE or the default in manage.py": "作为 --settings 传入;留空则沿用 DJANGO_SETTINGS_MODULE 或 manage.py 中的默认值",
"e.g. --keepdb --parallel": "例如 --keepdb --parallel",
"select files in a git repository to shelve": "请选择 Git 仓库中的文件以搁置更改",
- "select files from a single git repository to shelve": "请一次只选择同一个 Git 仓库中的文件以搁置更改"
+ "select files from a single git repository to shelve": "请一次只选择同一个 Git 仓库中的文件以搁置更改",
+ "External file changes sync may be slow": "外部文件更改同步可能较慢"
}