perf(vfs): share nested repo GitignoreStacks and bound synchronous watch_tree startup (#1705)

Opening a workspace with nested repositories, or a large repository on Linux, could stall open_project for minutes. GitignoreStack cached nested repositories per parent directory and cleared that cache on every created or removed path, so the startup walk rediscovered and recompiled the same nested repository's ignore rules for each directory; the per-directory watch walk also ran to completion on the open_project thread.

Nested stacks are now cached by repository root and invalidated only under the changed path (a `.git` appearing or disappearing included). The startup walk registers up to 256 directories synchronously and hands the rest to the watcher thread; children of a verified directory are matched by their own name only.

Tests cover the deferred startup watches and a repository created inside the workspace becoming an ignore boundary.

Co-authored-by: dengliu <[email protected]>

crates/ide-vfs/src/rules.rs

@@ -81,6 +81,14 @@ fn exclude_path(root: &Path) -> PathBuf {
         .unwrap_or_else(|| root.join(".git/info/exclude"))
 }
 
+#[derive(Default)]
+struct NestedRepos {
+    /// Directory -> enclosing repository root when different from `GitignoreStack::repo_root`.
+    by_dir: HashMap<PathBuf, Option<PathBuf>>,
+    /// Repository root -> shared `GitignoreStack` for that nested repository.
+    by_root: HashMap<PathBuf, Arc<GitignoreStack>>,
+}
+
 /// Layered gitignore matching: per-directory `.gitignore` / `.ignore` from the
 /// path's directory up to the repository root (or the file system root when
 /// not in a repository), then `.git/info/exclude` and the global excludes file.
@@ -89,7 +97,7 @@ pub(crate) struct GitignoreStack {
     /// `.git/info/exclude` and the global excludes file, rooted at the repo root.
     repo_level: RwLock<Vec<Gitignore>>,
     cache: Mutex<HashMap<PathBuf, Option<Arc<Gitignore>>>>,
-    nested: Mutex<HashMap<PathBuf, Option<Arc<GitignoreStack>>>>,
+    nested: Mutex<NestedRepos>,
 }
 
 /// How long [`GitignoreStack::cached`] reuses the repository lookup of a directory.
@@ -191,13 +199,23 @@ impl GitignoreStack {
         if let (Some(root), Some(map)) = (&self.repo_root, REPO_LEVEL.lock().as_mut()) {
             map.retain(|dir, _| !dir.starts_with(root));
         }
-        self.nested.lock().clear();
+        self.nested.lock().by_root.values().for_each(|s| s.reload_repo_level());
         *self.repo_level.write() = Self::load_repo_level(self.repo_root.as_deref());
     }
 
     /// Forgets the cached rules of `dir` and every directory below it.
     pub fn invalidate_tree(&self, dir: &Path) {
-        self.nested.lock().clear();
+        {
+            let mut nested = self.nested.lock();
+            let target = if dir.file_name().is_some_and(|n| n == ".git") { dir.parent().unwrap_or(dir) } else { dir };
+            if nested.by_dir.contains_key(target) || nested.by_root.contains_key(target) {
+                nested.by_dir.retain(|k, _| !k.starts_with(target));
+                nested.by_root.retain(|k, _| !k.starts_with(target));
+            }
+            for stack in nested.by_root.values() {
+                stack.invalidate_tree(dir);
+            }
+        }
         let mut cache = self.cache.lock();
         // Lookups always populate every ancestor, so no key below `dir` exists without `dir`.
         if cache.contains_key(dir) {
@@ -211,10 +229,57 @@ impl GitignoreStack {
 
     /// Forgets the cached rules of `dir` (after its `.gitignore` changed).
     pub fn invalidate(&self, dir: &Path) {
-        self.nested.lock().clear();
+        for stack in self.nested.lock().by_root.values() {
+            stack.invalidate(dir);
+        }
         self.cache.lock().remove(dir);
     }
 
+    /// Returns the shared `GitignoreStack` of the nested repository enclosing `dir`, if any.
+    fn nested_for(&self, dir: &Path) -> Option<Arc<GitignoreStack>> {
+        let mut nested = self.nested.lock();
+        let root = if let Some(cached) = nested.by_dir.get(dir) {
+            cached.clone()
+        } else {
+            let mut chain = Vec::new();
+            let mut found = None;
+            for anc in dir.ancestors() {
+                if let Some(cached) = nested.by_dir.get(anc) {
+                    found = cached.clone();
+                    break;
+                }
+                chain.push(anc.to_path_buf());
+                if self.repo_root.as_deref() == Some(anc) {
+                    found = None;
+                    break;
+                }
+                if anc.join(".git").exists() {
+                    found = Some(anc.to_path_buf());
+                    break;
+                }
+            }
+            for p in chain {
+                nested.by_dir.insert(p, found.clone());
+            }
+            found
+        }?;
+        Some(
+            nested
+                .by_root
+                .entry(root.clone())
+                .or_insert_with(|| {
+                    let repo_level = RwLock::new(Self::load_repo_level(Some(&root)));
+                    Arc::new(Self {
+                        repo_root: Some(root),
+                        repo_level,
+                        cache: Mutex::default(),
+                        nested: Mutex::default(),
+                    })
+                })
+                .clone(),
+        )
+    }
+
     fn matcher_for(&self, dir: &Path) -> Option<Arc<Gitignore>> {
         if let Some(cached) = self.cache.lock().get(dir) {
             return cached.clone();
@@ -250,20 +315,10 @@ impl GitignoreStack {
     pub fn is_ignored(&self, path: &Path, is_dir: bool) -> bool {
         // A project's watcher can span independent nested repositories. Their local
         // excludes and ignore ancestry must stop at their own repository boundary.
-        if let Some(parent) = path.parent() {
-            let nested = self
-                .nested
-                .lock()
-                .entry(parent.to_path_buf())
-                .or_insert_with(|| {
-                    find_repo_root(parent)
-                        .filter(|root| self.repo_root.as_ref() != Some(root))
-                        .map(|_| Arc::new(Self::new(parent)))
-                })
-                .clone();
-            if let Some(nested) = nested {
-                return nested.is_ignored(path, is_dir);
-            }
+        if let Some(parent) = path.parent()
+            && let Some(nested) = self.nested_for(parent)
+        {
+            return nested.is_ignored(path, is_dir);
         }
         if self.repo_root.as_deref() == Some(path) {
             return false;
@@ -302,20 +357,10 @@ impl GitignoreStack {
     /// ignored) still emit its own create/remove so the tree can show it
     /// appearing marked as ignored, while files inside it stay filtered out.
     pub fn is_ignored_self(&self, path: &Path, is_dir: bool) -> bool {
-        if let Some(parent) = path.parent() {
-            let nested = self
-                .nested
-                .lock()
-                .entry(parent.to_path_buf())
-                .or_insert_with(|| {
-                    find_repo_root(parent)
-                        .filter(|root| self.repo_root.as_ref() != Some(root))
-                        .map(|_| Arc::new(Self::new(parent)))
-                })
-                .clone();
-            if let Some(nested) = nested {
-                return nested.is_ignored_self(path, is_dir);
-            }
+        if let Some(parent) = path.parent()
+            && let Some(nested) = self.nested_for(parent)
+        {
+            return nested.is_ignored_self(path, is_dir);
         }
         if self.repo_root.as_deref() == Some(path) {
             return false;
@@ -405,4 +450,49 @@ mod tests {
         assert_eq!(first.repo_root(), again.repo_root());
         assert_eq!(again.repo_root(), Some(tmp.path()));
     }
+
+    #[test]
+    fn nested_repository_shares_one_gitignore_stack_across_subdirectories() {
+        let tmp = tempfile::tempdir().unwrap();
+        let workspace = tmp.path();
+        let repo = workspace.join("yolo");
+        std::fs::create_dir_all(repo.join(".git")).unwrap();
+        std::fs::create_dir_all(repo.join("a/b/c")).unwrap();
+        std::fs::create_dir_all(repo.join("x/y/z")).unwrap();
+        std::fs::write(repo.join(".gitignore"), "*.ignored\n").unwrap();
+
+        let stack = GitignoreStack::new(workspace);
+        assert!(stack.is_ignored(&repo.join("a/b/c/file.ignored"), false));
+        assert!(stack.is_ignored(&repo.join("x/y/z/other.ignored"), false));
+        assert!(!stack.is_ignored(&repo.join("x/y/z/keep.rs"), false));
+
+        assert_eq!(stack.nested.lock().by_root.len(), 1);
+        let nested = stack.nested.lock().by_root.get(&repo).unwrap().clone();
+        assert!(nested.cache.lock().contains_key(&repo));
+
+        // Invalidating a leaf file or directory inside the nested repo keeps its shared stack.
+        stack.invalidate_tree(&repo.join("a/b/c/new.rs"));
+        assert_eq!(stack.nested.lock().by_root.len(), 1);
+        assert!(Arc::ptr_eq(&nested, stack.nested.lock().by_root.get(&repo).unwrap()));
+    }
+
+    #[test]
+    fn a_repository_created_inside_the_workspace_becomes_a_boundary() {
+        let tmp = tempfile::tempdir().unwrap();
+        let workspace = tmp.path();
+        std::fs::create_dir_all(workspace.join(".git")).unwrap();
+        std::fs::create_dir_all(workspace.join("sub/deep")).unwrap();
+        std::fs::write(workspace.join(".gitignore"), "*.tmp\n").unwrap();
+
+        let stack = GitignoreStack::new(workspace);
+        assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+
+        std::fs::create_dir(workspace.join("sub/.git")).unwrap();
+        stack.invalidate_tree(&workspace.join("sub/.git"));
+        assert!(!stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+
+        std::fs::remove_dir(workspace.join("sub/.git")).unwrap();
+        stack.invalidate_tree(&workspace.join("sub/.git"));
+        assert!(stack.is_ignored(&workspace.join("sub/deep/a.tmp"), false));
+    }
 }

crates/ide-vfs/src/watch.rs

@@ -27,6 +27,9 @@ const DEBOUNCE: Duration = Duration::from_millis(100);
 const MAX_DELAY: Duration = Duration::from_millis(1000);
 /// How often the polling fallback rescans the watched directories.
 const POLL_INTERVAL: Duration = Duration::from_secs(2);
+/// Directories watched synchronously in [`Watcher::start`] before handing the
+/// remainder of a large tree to the watcher thread.
+const SYNC_WATCH_BUDGET: usize = 256;
 
 /// How directories are registered with the OS.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -178,11 +181,12 @@ impl Watcher {
         )
         .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?;
         *state.watcher.lock() = Some(Box::new(watcher));
-        if mode == WatchMode::PerDirectory {
-            state.watch_tree(root, false, &mut Vec::new());
+        let deferred = if mode == WatchMode::PerDirectory {
+            state.watch_tree_bounded(root, SYNC_WATCH_BUDGET)
         } else {
             state.add_watch(root);
-        }
+            Vec::new()
+        };
         if state.watched.lock().is_empty() {
             return Err(io_error(format!("cannot watch {}", root.display())));
         }
@@ -190,7 +194,7 @@ impl Watcher {
             let state = state.clone();
             std::thread::Builder::new()
                 .name("openidea-fs-watch".into())
-                .spawn(move || state.run(&*emit))
+                .spawn(move || state.run(deferred, &*emit))
                 .map_err(|e| io_error(format!("cannot start file watcher: {e}")))?
         };
         Ok(Inner { state, thread: Some(thread) })
@@ -239,8 +243,12 @@ impl DirState {
         self.stop.load(Ordering::SeqCst)
     }
 
-    /// Debounce loop: waits for a quiet period, then processes the batch.
-    fn run(&self, emit: impl Fn(&FsBatch)) {
+    /// Debounce loop: finishes any deferred startup watches, then waits for a
+    /// quiet period and processes each batch.
+    fn run(&self, deferred: Vec<(PathBuf, bool)>, emit: impl Fn(&FsBatch)) {
+        if !deferred.is_empty() {
+            self.watch_stack(deferred, false, &mut Vec::new(), None);
+        }
         loop {
             let events = {
                 let mut q = self.queue.lock();
@@ -298,6 +306,7 @@ impl DirState {
         // during this batch would otherwise never be seen.
         for path in &batch.vcs {
             if !self.watched.lock().contains(path) && path.symlink_metadata().is_ok_and(|m| m.is_dir()) {
+                self.filter.gitignore.invalidate_tree(path);
                 self.watch_tree(path, false, &mut discovered);
             }
         }
@@ -349,11 +358,33 @@ impl DirState {
     /// files and directories found inside (created before the watch was in
     /// place) are appended to `found`.
     fn watch_tree(&self, dir: &Path, report: bool, found: &mut Vec<PathBuf>) {
-        let mut stack = vec![dir.to_path_buf()];
-        while let Some(dir) = stack.pop() {
-            if self.stopped() || !self.filter.should_watch_dir(&dir) || !self.add_watch(&dir) {
+        self.watch_stack(vec![(dir.to_path_buf(), false)], report, found, None);
+    }
+
+    /// Watches up to `budget` directories synchronously and returns any
+    /// remaining `(dir, parent_verified)` entries for background registration.
+    fn watch_tree_bounded(&self, dir: &Path, budget: usize) -> Vec<(PathBuf, bool)> {
+        self.watch_stack(vec![(dir.to_path_buf(), false)], false, &mut Vec::new(), Some(budget))
+    }
+
+    fn watch_stack(
+        &self,
+        mut stack: Vec<(PathBuf, bool)>,
+        report: bool,
+        found: &mut Vec<PathBuf>,
+        budget: Option<usize>,
+    ) -> Vec<(PathBuf, bool)> {
+        let mut watched_now = 0;
+        while let Some((dir, parent_ok)) = stack.pop() {
+            if self.stopped() {
+                break;
+            }
+            let should_watch =
+                if parent_ok { self.filter.should_watch_child_dir(&dir) } else { self.filter.should_watch_dir(&dir) };
+            if !should_watch || !self.add_watch(&dir) {
                 continue;
             }
+            watched_now += 1;
             let Ok(entries) = std::fs::read_dir(&dir) else { continue };
             for entry in entries.flatten() {
                 let path = entry.path();
@@ -362,10 +393,14 @@ impl DirState {
                     found.push(path.clone());
                 }
                 if is_dir {
-                    stack.push(path);
+                    stack.push((path, true));
                 }
             }
+            if budget.is_some_and(|max| watched_now >= max) {
+                break;
+            }
         }
+        stack
     }
 
     fn unwatch_tree(&self, dir: &Path) {
@@ -469,6 +504,15 @@ impl EventFilter {
     /// and `worktrees/` (HEAD, index, refs and worktree metadata are what VCS
     /// refresh needs).
     pub fn should_watch_dir(&self, dir: &Path) -> bool {
+        self.should_watch_dir_inner(dir, false)
+    }
+
+    /// Fast path when `dir`'s parent directory is already known to be watchable.
+    fn should_watch_child_dir(&self, dir: &Path) -> bool {
+        self.should_watch_dir_inner(dir, true)
+    }
+
+    fn should_watch_dir_inner(&self, dir: &Path, parent_ok: bool) -> bool {
         let Ok(rel) = dir.strip_prefix(&self.root) else { return false };
         if rel.as_os_str().is_empty() {
             return true;
@@ -484,7 +528,11 @@ impl EventFilter {
                 _ => false,
             };
         }
-        !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)
+        if parent_ok {
+            !self.excludes.read().is_match(rel) && !self.gitignore.is_ignored_self(dir, true)
+        } else {
+            !self.excludes.read().is_excluded(rel) && !self.gitignore.is_ignored(dir, true)
+        }
     }
 
     /// Whether a path found by scanning a new directory would be reported.
@@ -1075,6 +1123,42 @@ mod tests {
         assert_eq!(batches.lock().len(), n, "no events after drop");
     }
 
+    #[test]
+    fn per_directory_mode_watches_the_rest_of_a_large_tree_after_start() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = dunce::canonicalize(tmp.path()).unwrap();
+        let dirs = SYNC_WATCH_BUDGET + 40;
+        for i in 0..dirs {
+            std::fs::create_dir_all(root.join(format!("d{i:03}"))).unwrap();
+        }
+        std::fs::create_dir_all(root.join("node_modules/x")).unwrap();
+        let batches: Arc<Mutex<Vec<FsBatch>>> = Arc::default();
+        let sink: SharedSink = Arc::new(|_e: IdeEvent| {});
+        let watcher = {
+            let batches = batches.clone();
+            Watcher::start_with(
+                &root,
+                vec!["**/node_modules".into()],
+                sink,
+                Box::new(move |b| batches.lock().push(b.clone())),
+                WatchMode::PerDirectory,
+            )
+            .unwrap()
+        };
+        let deadline = Instant::now() + Duration::from_secs(10);
+        while watcher.watch_count() < dirs + 1 && Instant::now() < deadline {
+            std::thread::sleep(Duration::from_millis(20));
+        }
+        // root and every d*, but not node_modules
+        assert_eq!(watcher.watch_count(), dirs + 1);
+        std::thread::sleep(Duration::from_millis(200));
+        for i in [0, dirs - 1] {
+            let file = root.join(format!("d{i:03}/f.rs"));
+            std::fs::write(&file, "x").unwrap();
+            assert!(wait_for(&batches, |b| all_paths(b).contains(&file)), "{file:?}");
+        }
+    }
+
     #[test]
     fn per_directory_mode_follows_worktree_registrations() {
         let tmp = tempfile::tempdir().unwrap();