[ty] Speedup project file discovery (#19913)

Author: MichaReiser

crates/ruff_db/src/files.rs

@@ -87,11 +87,12 @@ impl Files {
             .system_by_path
             .entry(absolute.clone())
             .or_insert_with(|| {
-                tracing::trace!("Adding file '{path}'");
-
                 let metadata = db.system().path_metadata(path);
+
+                tracing::trace!("Adding file '{absolute}'");
+
                 let durability = self
-                    .root(db, path)
+                    .root(db, &absolute)
                     .map_or(Durability::default(), |root| root.durability(db));
 
                 let builder = File::builder(FilePath::System(absolute))

crates/ty_project/src/db.rs

@@ -21,6 +21,8 @@ mod changes;
 #[salsa::db]
 pub trait Db: SemanticDb {
     fn project(&self) -> Project;
+
+    fn dyn_clone(&self) -> Box<dyn Db>;
 }
 
 #[salsa::db]
@@ -484,6 +486,10 @@ impl Db for ProjectDatabase {
     fn project(&self) -> Project {
         self.project.unwrap()
     }
+
+    fn dyn_clone(&self) -> Box<dyn Db> {
+        Box::new(self.clone())
+    }
 }
 
 #[cfg(feature = "format")]
@@ -611,6 +617,10 @@ pub(crate) mod tests {
         fn project(&self) -> Project {
             self.project.unwrap()
         }
+
+        fn dyn_clone(&self) -> Box<dyn Db> {
+            Box::new(self.clone())
+        }
     }
 
     #[salsa::db]

crates/ty_project/src/walk.rs

@@ -4,7 +4,7 @@ use ruff_db::files::{File, system_path_to_file};
 use ruff_db::system::walk_directory::{ErrorKind, WalkDirectoryBuilder, WalkState};
 use ruff_db::system::{SystemPath, SystemPathBuf};
 use ruff_python_ast::PySourceType;
-use rustc_hash::{FxBuildHasher, FxHashSet};
+use rustc_hash::FxHashSet;
 use std::path::PathBuf;
 use thiserror::Error;
 
@@ -163,20 +163,24 @@ impl<'a> ProjectFilesWalker<'a> {
 
     /// Walks the project paths and collects the paths of all files that
     /// are included in the project.
-    pub(crate) fn walk_paths(self) -> (Vec<SystemPathBuf>, Vec<IOErrorDiagnostic>) {
-        let paths = std::sync::Mutex::new(Vec::new());
+    pub(crate) fn collect_vec(self, db: &dyn Db) -> (Vec<File>, Vec<IOErrorDiagnostic>) {
+        let files = std::sync::Mutex::new(Vec::new());
         let diagnostics = std::sync::Mutex::new(Vec::new());
 
         self.walker.run(|| {
-            Box::new(|entry| {
+            let db = db.dyn_clone();
+            let filter = &self.filter;
+            let files = &files;
+            let diagnostics = &diagnostics;
+
+            Box::new(move |entry| {
                 match entry {
                     Ok(entry) => {
                         // Skip excluded directories unless they were explicitly passed to the walker
                         // (which is the case passed to `ty check <paths>`).
                         if entry.file_type().is_directory() {
                             if entry.depth() > 0 {
-                                let directory_included = self
-                                    .filter
+                                let directory_included = filter
                                     .is_directory_included(entry.path(), GlobFilterCheckMode::TopDown);
                                 return match directory_included {
                                     IncludeResult::Included => WalkState::Continue,
@@ -210,8 +214,7 @@ impl<'a> ProjectFilesWalker<'a> {
                             // For all files, except the ones that were explicitly passed to the walker (CLI),
                             // check if they're included in the project.
                             if entry.depth() > 0 {
-                                match self
-                                    .filter
+                                match filter
                                     .is_file_included(entry.path(), GlobFilterCheckMode::TopDown)
                                 {
                                     IncludeResult::Included => {},
@@ -232,8 +235,11 @@ impl<'a> ProjectFilesWalker<'a> {
                                 }
                             }
 
-                            let mut paths = paths.lock().unwrap();
-                            paths.push(entry.into_path());
+                            // If this returns `Err`, then the file was deleted between now and when the walk callback was called.
+                            // We can ignore this.
+                            if let Ok(file) = system_path_to_file(&*db, entry.path()) {
+                                files.lock().unwrap().push(file);
+                            }
                         }
                     }
                     Err(error) => match error.kind() {
@@ -274,39 +280,14 @@ impl<'a> ProjectFilesWalker<'a> {
         });
 
         (
-            paths.into_inner().unwrap(),
+            files.into_inner().unwrap(),
             diagnostics.into_inner().unwrap(),
         )
     }
 
-    pub(crate) fn collect_vec(self, db: &dyn Db) -> (Vec<File>, Vec<IOErrorDiagnostic>) {
-        let (paths, diagnostics) = self.walk_paths();
-
-        (
-            paths
-                .into_iter()
-                .filter_map(move |path| {
-                    // If this returns `None`, then the file was deleted between the `walk_directory` call and now.
-                    // We can ignore this.
-                    system_path_to_file(db, &path).ok()
-                })
-                .collect(),
-            diagnostics,
-        )
-    }
-
     pub(crate) fn collect_set(self, db: &dyn Db) -> (FxHashSet<File>, Vec<IOErrorDiagnostic>) {
-        let (paths, diagnostics) = self.walk_paths();
-
-        let mut files = FxHashSet::with_capacity_and_hasher(paths.len(), FxBuildHasher);
-
-        for path in paths {
-            if let Ok(file) = system_path_to_file(db, &path) {
-                files.insert(file);
-            }
-        }
-
-        (files, diagnostics)
+        let (files, diagnostics) = self.collect_vec(db);
+        (files.into_iter().collect(), diagnostics)
     }
 }