[ty] Speedup project file discovery

crates/ruff_db/src/files.rs

@@ -87,33 +87,70 @@ impl Files {
             .system_by_path
             .entry(absolute.clone())
             .or_insert_with(|| {
-                tracing::trace!("Adding file '{path}'");
-
                 let metadata = db.system().path_metadata(path);
-                let durability = self
-                    .root(db, path)
-                    .map_or(Durability::default(), |root| root.durability(db));
-
-                let builder = File::builder(FilePath::System(absolute))
-                    .durability(durability)
-                    .path_durability(Durability::HIGH);
-
-                let builder = match metadata {
-                    Ok(metadata) if metadata.file_type().is_file() => builder
-                        .permissions(metadata.permissions())
-                        .revision(metadata.revision()),
-                    Ok(metadata) if metadata.file_type().is_directory() => {
-                        builder.status(FileStatus::IsADirectory)
-                    }
-                    _ => builder
-                        .status(FileStatus::NotFound)
-                        .status_durability(Durability::MEDIUM.max(durability)),
-                };
 
-                builder.new(db)
+                self.create_system_entry(db, absolute, metadata)
             })
     }
 
+    /// Looks up or inserts a [`File`] at the given path using the provided metadata.
+    ///
+    ///
+    /// Prefer using [`system_path_to_file`] unless its `stats` call when inserting a new file is a performance
+    /// concern because the metadata is already available or when computing the metadata on different threads.
+    ///
+    /// The metadata is only used when inserting a new file. It doesn't
+    /// update the metadata if a [`File`] for `path` already exists.
+    ///
+    /// This function differs from [`system_path_to_file`] in that it doesn't add a dependency
+    /// on `File.status` and always returns a `File`, even
+    /// if `path` doesn't point to a file (e.g. if `path` is a directory).
+    pub fn system_from_metadata(
+        &self,
+        db: &dyn Db,
+        path: &SystemPath,
+        metadata: crate::system::Metadata,
+    ) -> File {
+        let absolute = SystemPath::absolute(path, db.system().current_directory());
+
+        *self
+            .inner
+            .system_by_path
+            .entry(absolute.clone())
+            .or_insert_with(|| self.create_system_entry(db, absolute, Ok(metadata)))
+    }
+
+    fn create_system_entry(
+        &self,
+        db: &dyn Db,
+        absolute_path: SystemPathBuf,
+        metadata: crate::system::Result<crate::system::Metadata>,
+    ) -> File {
+        tracing::trace!("Adding file '{absolute_path}'");
+
+        let durability = self
+            .root(db, &absolute_path)
+            .map_or(Durability::default(), |root| root.durability(db));
+
+        let builder = File::builder(FilePath::System(absolute_path))
+            .durability(durability)
+            .path_durability(Durability::HIGH);
+
+        let builder = match metadata {
+            Ok(metadata) if metadata.file_type().is_file() => builder
+                .permissions(metadata.permissions())
+                .revision(metadata.revision()),
+            Ok(metadata) if metadata.file_type().is_directory() => {
+                builder.status(FileStatus::IsADirectory)
+            }
+            _ => builder
+                .status(FileStatus::NotFound)
+                .status_durability(Durability::MEDIUM.max(durability)),
+        };
+
+        builder.new(db)
+    }
+
     /// Tries to look up the file for the given system path, returns `None` if no such file exists yet
     pub fn try_system(&self, db: &dyn Db, path: &SystemPath) -> Option<File> {
         let absolute = SystemPath::absolute(path, db.system().current_directory());

crates/ruff_db/src/system.rs

@@ -46,7 +46,7 @@ pub type Result<T> = std::io::Result<T>;
 ///    * File watching isn't supported.
 ///
 /// Abstracting the system also enables tests to use a more efficient in-memory file system.
-pub trait System: Debug {
+pub trait System: Debug + Sync {
     /// Reads the metadata of the file or directory at `path`.
     ///
     /// This function will traverse symbolic links to query information about the destination file.

crates/ty_project/src/walk.rs

@@ -1,10 +1,10 @@
 use crate::glob::IncludeExcludeFilter;
 use crate::{Db, GlobFilterCheckMode, IOErrorDiagnostic, IOErrorKind, IncludeResult, Project};
-use ruff_db::files::{File, system_path_to_file};
+use ruff_db::files::File;
 use ruff_db::system::walk_directory::{ErrorKind, WalkDirectoryBuilder, WalkState};
-use ruff_db::system::{SystemPath, SystemPathBuf};
+use ruff_db::system::{Metadata, SystemPath, SystemPathBuf};
 use ruff_python_ast::PySourceType;
-use rustc_hash::{FxBuildHasher, FxHashSet};
+use rustc_hash::FxHashSet;
 use std::path::PathBuf;
 use thiserror::Error;
 
@@ -163,10 +163,15 @@ impl<'a> ProjectFilesWalker<'a> {
 
     /// Walks the project paths and collects the paths of all files that
     /// are included in the project.
-    pub(crate) fn walk_paths(self) -> (Vec<SystemPathBuf>, Vec<IOErrorDiagnostic>) {
+    pub(crate) fn walk_paths(
+        self,
+        db: &dyn Db,
+    ) -> (Vec<(SystemPathBuf, Metadata)>, Vec<IOErrorDiagnostic>) {
         let paths = std::sync::Mutex::new(Vec::new());
         let diagnostics = std::sync::Mutex::new(Vec::new());
 
+        let system = db.system();
+
         self.walker.run(|| {
             Box::new(|entry| {
                 match entry {
@@ -232,8 +237,12 @@ impl<'a> ProjectFilesWalker<'a> {
                                 }
                             }
 
-                            let mut paths = paths.lock().unwrap();
-                            paths.push(entry.into_path());
+                            // If this returns `Err`, then the file was deleted between now and when the walk callback was called.
+                            // We can ignore this.
+                            if let Ok(metadata) = system.path_metadata(entry.path()) {
+                                let mut paths = paths.lock().unwrap();
+                                paths.push((entry.into_path(), metadata));
+                            }
                         }
                     }
                     Err(error) => match error.kind() {
@@ -280,31 +289,24 @@ impl<'a> ProjectFilesWalker<'a> {
     }
 
     pub(crate) fn collect_vec(self, db: &dyn Db) -> (Vec<File>, Vec<IOErrorDiagnostic>) {
-        let (paths, diagnostics) = self.walk_paths();
+        let (paths, diagnostics) = self.walk_paths(db);
 
         (
             paths
                 .into_iter()
-                .filter_map(move |path| {
-                    // If this returns `None`, then the file was deleted between the `walk_directory` call and now.
-                    // We can ignore this.
-                    system_path_to_file(db, &path).ok()
-                })
+                .map(|(path, metadata)| db.files().system_from_metadata(db, &path, metadata))
                 .collect(),
             diagnostics,
         )
     }
 
     pub(crate) fn collect_set(self, db: &dyn Db) -> (FxHashSet<File>, Vec<IOErrorDiagnostic>) {
-        let (paths, diagnostics) = self.walk_paths();
-
-        let mut files = FxHashSet::with_capacity_and_hasher(paths.len(), FxBuildHasher);
+        let (paths, diagnostics) = self.walk_paths(db);
 
-        for path in paths {
-            if let Ok(file) = system_path_to_file(db, &path) {
-                files.insert(file);
-            }
-        }
+        let files: FxHashSet<_> = paths
+            .into_iter()
+            .map(|(path, metadata)| db.files().system_from_metadata(db, &path, metadata))
+            .collect();
 
         (files, diagnostics)
     }