[heft] Add options to only resolve symlinks that are within node_modules (#5011)

dmichon-msft · web-flow · commit 6f5f09d8b484 · 2024-11-21T14:39:48.000-08:00
* [node-core-library] Add RealNodeModulePathResolver

* [heft-typescript] Add onlyResolveSymlinksInNodeModules option

* [heft-jest] Add node module symlink resolver

* Test resolver

* Use features in local-node-rig

* Fix duplicate types

* Disable CI fail-fast

* Pay extra lstat

* Fix unit tests

---------

Co-authored-by: David Michon &lt;dmichon-msft@users.noreply.github.com&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,6 +9,7 @@ on:
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
         include:
           - NodeVersion: 18.18.x
diff --git a/build-tests/heft-webpack5-everything-test/config/jest.config.json b/build-tests/heft-webpack5-everything-test/config/jest.config.json
@@ -7,5 +7,6 @@
   "coverageReporters": ["cobertura", "html"],
 
   // Use v8 coverage provider to avoid Babel
-  "coverageProvider": "v8"
+  "coverageProvider": "v8",
+  "resolver": "@rushstack/heft-jest-plugin/lib/exports/jest-node-modules-symlink-resolver"
 }
diff --git a/build-tests/heft-webpack5-everything-test/config/typescript.json b/build-tests/heft-webpack5-everything-test/config/typescript.json
@@ -49,5 +49,7 @@
     // "excludeGlobs": [
     //   "some/path/*.css"
     // ]
-  }
+  },
+
+  "onlyResolveSymlinksInNodeModules": true
 }
diff --git a/common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json b/common/changes/@rushstack/heft-jest-plugin/fast-realpath_2024-11-20-05-21.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/heft-jest-plugin",
+      "comment": "Add a custom resolver that only resolves symlinks that are within node_modules.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/heft-jest-plugin"
+}
diff --git a/common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json b/common/changes/@rushstack/heft-typescript-plugin/fast-realpath_2024-11-20-04-20.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/heft-typescript-plugin",
+      "comment": "Add \"onlyResolveSymlinksInNodeModules\" option to improve performance for typical repository layouts.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/heft-typescript-plugin"
+}
diff --git a/common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json b/common/changes/@rushstack/node-core-library/fast-realpath_2024-11-20-19-57.json
@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/node-core-library",
+      "comment": "Add `RealNodeModulePathResolver` class to get equivalent behavior to `realpath` with fewer system calls (and therefore higher performance) in the typical scenario where the only symlinks in the repository are inside of `node_modules` folders and are links to package folders.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/node-core-library"
+}
diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json
@@ -230,6 +230,10 @@
       "name": "@rushstack/package-extractor",
       "allowedCategories": [ "libraries", "vscode-extensions" ]
     },
+    {
+      "name": "@rushstack/real-node-module-path",
+      "allowedCategories": [ "libraries" ]
+    },
     {
       "name": "@rushstack/rig-package",
       "allowedCategories": [ "libraries" ]
diff --git a/common/reviews/api/heft-typescript-plugin.api.md b/common/reviews/api/heft-typescript-plugin.api.md
@@ -55,6 +55,7 @@ export interface ITypeScriptConfigurationJson {
     buildProjectReferences?: boolean;
     emitCjsExtensionForCommonJS?: boolean | undefined;
     emitMjsExtensionForESModule?: boolean | undefined;
+    onlyResolveSymlinksInNodeModules?: boolean;
     // (undocumented)
     project?: string;
     staticAssetsToCopy?: IStaticAssetsCopyConfiguration;
diff --git a/common/reviews/api/node-core-library.api.md b/common/reviews/api/node-core-library.api.md
@@ -7,7 +7,8 @@
 /// <reference types="node" />
 
 import * as child_process from 'child_process';
-import * as fs from 'fs';
+import * as nodeFs from 'fs';
+import * as nodePath from 'path';
 
 // @public
 export enum AlreadyExistsBehavior {
@@ -213,7 +214,7 @@ export type FileSystemCopyFilesAsyncFilter = (sourcePath: string, destinationPat
 export type FileSystemCopyFilesFilter = (sourcePath: string, destinationPath: string) => boolean;
 
 // @public
-export type FileSystemStats = fs.Stats;
+export type FileSystemStats = nodeFs.Stats;
 
 // @public
 export class FileWriter {
@@ -231,7 +232,7 @@ export const FolderConstants: {
 };
 
 // @public
-export type FolderItem = fs.Dirent;
+export type FolderItem = nodeFs.Dirent;
 
 // @public
 export interface IAsyncParallelismOptions {
@@ -605,6 +606,14 @@ export interface IReadLinesFromIterableOptions {
     ignoreEmptyLines?: boolean;
 }
 
+// @public
+export interface IRealNodeModulePathResolverOptions {
+    // (undocumented)
+    fs: Pick<typeof nodeFs, 'lstatSync' | 'readlinkSync'>;
+    // (undocumented)
+    path: Pick<typeof nodePath, 'isAbsolute' | 'normalize' | 'resolve' | 'sep'>;
+}
+
 // @public (undocumented)
 export interface IRunWithRetriesOptions<TResult> {
     // (undocumented)
@@ -834,6 +843,13 @@ export class ProtectableMap<K, V> {
     get size(): number;
 }
 
+// @public
+export class RealNodeModulePathResolver {
+    constructor(options?: IRealNodeModulePathResolverOptions);
+    clearCache(): void;
+    readonly realNodeModulePath: (input: string) => string;
+}
+
 // @public
 export class Sort {
     static compareByValue(x: any, y: any): number;
diff --git a/common/reviews/api/real-node-module-path.api.md b/common/reviews/api/real-node-module-path.api.md
@@ -0,0 +1,17 @@
+## API Report File for "@rushstack/real-node-module-path"
+
+> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
+
+```ts
+
+/// <reference types="node" />
+
+// @public
+export function clearCache(): void;
+
+// @public
+export const realNodeModulePath: (input: string) => string;
+
+// (No @packageDocumentation comment for this package)
+
+```
diff --git a/heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts b/heft-plugins/heft-jest-plugin/src/JestRealPathPatch.ts
@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as path from 'node:path';
+import { RealNodeModulePathResolver } from '@rushstack/node-core-library/lib/RealNodeModulePath';
+
+const jestResolvePackageFolder: string = path.dirname(require.resolve('jest-resolve/package.json'));
+const jestResolveFileWalkersPath: string = path.resolve(jestResolvePackageFolder, './build/fileWalkers.js');
+
+const jestUtilPackageFolder: string = path.dirname(
+  require.resolve('jest-util/package.json', { paths: [jestResolvePackageFolder] })
+);
+const jestUtilTryRealpathPath: string = path.resolve(jestUtilPackageFolder, './build/tryRealpath.js');
+
+const { realNodeModulePath }: RealNodeModulePathResolver = new RealNodeModulePathResolver();
+
+const fileWalkersModule: {
+  realpathSync: (filePath: string) => string;
+} = require(jestResolveFileWalkersPath);
+fileWalkersModule.realpathSync = realNodeModulePath;
+
+const tryRealpathModule: {
+  default: (filePath: string) => string;
+} = require(jestUtilTryRealpathPath);
+tryRealpathModule.default = (input: string): string => {
+  try {
+    return realNodeModulePath(input);
+  } catch (error) {
+    // Not using the helper from FileSystem here because this code loads in every Jest worker process
+    // and FileSystem has a lot of extra dependencies
+    if (error.code !== 'ENOENT' && error.code !== 'EISDIR') {
+      throw error;
+    }
+  }
+  return input;
+};
diff --git a/heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts b/heft-plugins/heft-jest-plugin/src/exports/jest-node-modules-symlink-resolver.ts
@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import '../JestRealPathPatch';
+// Using this syntax because HeftJestResolver uses `export =` syntax.
+import resolver = require('../HeftJestResolver');
+export = resolver;
diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptBuilder.ts
@@ -7,14 +7,21 @@ import { Worker } from 'worker_threads';
 
 import * as semver from 'semver';
 import type * as TTypescript from 'typescript';
-import { JsonFile, type IPackageJson, Path, FileError } from '@rushstack/node-core-library';
+import {
+  JsonFile,
+  type IPackageJson,
+  Path,
+  FileError,
+  RealNodeModulePathResolver
+} from '@rushstack/node-core-library';
 import type { ITerminal } from '@rushstack/terminal';
 import type { IScopedLogger } from '@rushstack/heft';
 
 import type {
   ExtendedBuilderProgram,
   ExtendedTypeScript,
-  IExtendedSolutionBuilder
+  IExtendedSolutionBuilder,
+  ITypeScriptNodeSystem
 } from './internalTypings/TypeScriptInternals';
 import type { ITypeScriptConfigurationJson } from './TypeScriptPlugin';
 import type { PerformanceMeasurer } from './Performance';
@@ -314,14 +321,41 @@ export class TypeScriptBuilder {
         return timeout;
       };
 
+      let realpath: typeof ts.sys.realpath = ts.sys.realpath;
+      if (this._configuration.onlyResolveSymlinksInNodeModules) {
+        const resolver: RealNodeModulePathResolver = new RealNodeModulePathResolver();
+        realpath = resolver.realNodeModulePath;
+      }
+
+      const getCurrentDirectory: () => string = () => this._configuration.buildFolderPath;
+
       // Need to also update watchFile and watchDirectory
-      const system: TTypescript.System = {
+      const system: ITypeScriptNodeSystem = {
         ...ts.sys,
-        getCurrentDirectory: () => this._configuration.buildFolderPath,
+        realpath,
+        getCurrentDirectory,
         clearTimeout,
         setTimeout
       };
 
+      if (realpath && system.getAccessibleFileSystemEntries) {
+        const { getAccessibleFileSystemEntries } = system;
+        system.readDirectory = (folderPath, extensions, exclude, include, depth): string[] => {
+          return ts.matchFiles(
+            folderPath,
+            extensions,
+            exclude,
+            include,
+            ts.sys.useCaseSensitiveFileNames,
+            getCurrentDirectory(),
+            depth,
+            getAccessibleFileSystemEntries,
+            realpath,
+            ts.sys.directoryExists
+          );
+        };
+      }
+
       this._tool = {
         ts,
         system,
@@ -376,7 +410,7 @@ export class TypeScriptBuilder {
     if (!tool.solutionBuilder && !tool.watchProgram) {
       //#region CONFIGURE
       const { duration: configureDurationMs, tsconfig } = measureTsPerformance('Configure', () => {
-        const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts);
+        const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool);
         this._validateTsconfig(ts, _tsconfig);
 
         return {
@@ -430,7 +464,7 @@ export class TypeScriptBuilder {
       tsconfig,
       compilerHost
     } = measureTsPerformance('Configure', () => {
-      const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts);
+      const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool);
       this._validateTsconfig(ts, _tsconfig);
 
       const _compilerHost: TTypescript.CompilerHost = this._buildIncrementalCompilerHost(tool, _tsconfig);
@@ -557,7 +591,7 @@ export class TypeScriptBuilder {
     if (!tool.solutionBuilder) {
       //#region CONFIGURE
       const { duration: configureDurationMs, solutionBuilderHost } = measureSync('Configure', () => {
-        const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(ts);
+        const _tsconfig: TTypescript.ParsedCommandLine = this._loadTsconfig(tool);
         this._validateTsconfig(ts, _tsconfig);
 
         const _solutionBuilderHost: TSolutionHost = this._buildSolutionBuilderHost(tool);
@@ -922,19 +956,21 @@ export class TypeScriptBuilder {
     return `${outFolderName}:${jsExtensionOverride || '.js'}`;
   }
 
-  private _loadTsconfig(ts: ExtendedTypeScript): TTypescript.ParsedCommandLine {
+  private _loadTsconfig(tool: ITypeScriptTool): TTypescript.ParsedCommandLine {
+    const { ts, system } = tool;
     const parsedConfigFile: ReturnType<typeof ts.readConfigFile> = ts.readConfigFile(
       this._configuration.tsconfigPath,
-      ts.sys.readFile
+      system.readFile
     );
 
     const currentFolder: string = path.dirname(this._configuration.tsconfigPath);
     const tsconfig: TTypescript.ParsedCommandLine = ts.parseJsonConfigFileContent(
       parsedConfigFile.config,
       {
-        fileExists: ts.sys.fileExists,
-        readFile: ts.sys.readFile,
-        readDirectory: ts.sys.readDirectory,
+        fileExists: system.fileExists,
+        readFile: system.readFile,
+        readDirectory: system.readDirectory,
+        realpath: system.realpath,
         useCaseSensitiveFileNames: true
       },
       currentFolder,
@@ -1054,11 +1090,11 @@ export class TypeScriptBuilder {
       // Do nothing
     };
 
-    const { ts } = tool;
+    const { ts, system } = tool;
 
     const solutionBuilderHost: TTypescript.SolutionBuilderHost<TTypescript.EmitAndSemanticDiagnosticsBuilderProgram> =
       ts.createSolutionBuilderHost(
-        ts.sys,
+        system,
         this._getCreateBuilderProgram(ts),
         tool.reportDiagnostic,
         reportSolutionBuilderStatus,
@@ -1090,7 +1126,11 @@ export class TypeScriptBuilder {
     if (tsconfig.options.incremental) {
       compilerHost = ts.createIncrementalCompilerHost(tsconfig.options, system);
     } else {
-      compilerHost = ts.createCompilerHost(tsconfig.options, undefined, system);
+      compilerHost = (ts.createCompilerHostWorker ?? ts.createCompilerHost)(
+        tsconfig.options,
+        undefined,
+        system
+      );
     }
 
     this._changeCompilerHostToUseCache(compilerHost, tool);
diff --git a/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts b/heft-plugins/heft-typescript-plugin/src/TypeScriptPlugin.ts
@@ -78,6 +78,12 @@ export interface ITypeScriptConfigurationJson {
    */
   useTranspilerWorker?: boolean;
 
+  /**
+   * If true, the TypeScript compiler will only resolve symlinks to their targets if the links are in a node_modules folder.
+   * This significantly reduces file system operations in typical usage.
+   */
+  onlyResolveSymlinksInNodeModules?: boolean;
+
   /*
    * Specifies the tsconfig.json file that will be used for compilation. Equivalent to the "project" argument for the 'tsc' and 'tslint' command line tools.
    *
@@ -365,6 +371,8 @@ export default class TypeScriptPlugin implements IHeftTaskPlugin {
 
       useTranspilerWorker: typeScriptConfigurationJson?.useTranspilerWorker,
 
+      onlyResolveSymlinksInNodeModules: typeScriptConfigurationJson?.onlyResolveSymlinksInNodeModules,
+
       tsconfigPath: getTsconfigFilePath(heftConfiguration, typeScriptConfigurationJson),
       additionalModuleKindsToEmit: typeScriptConfigurationJson?.additionalModuleKindsToEmit,
       emitCjsExtensionForCommonJS: !!typeScriptConfigurationJson?.emitCjsExtensionForCommonJS,
diff --git a/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts b/heft-plugins/heft-typescript-plugin/src/internalTypings/TypeScriptInternals.ts
diff --git a/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json b/heft-plugins/heft-typescript-plugin/src/schemas/typescript.schema.json
diff --git a/libraries/node-core-library/src/RealNodeModulePath.ts b/libraries/node-core-library/src/RealNodeModulePath.ts
diff --git a/libraries/node-core-library/src/index.ts b/libraries/node-core-library/src/index.ts
diff --git a/libraries/node-core-library/src/test/RealNodeModulePath.test.ts b/libraries/node-core-library/src/test/RealNodeModulePath.test.ts
diff --git a/rigs/local-node-rig/profiles/default/config/jest.config.json b/rigs/local-node-rig/profiles/default/config/jest.config.json
diff --git a/rigs/local-node-rig/profiles/default/config/typescript.json b/rigs/local-node-rig/profiles/default/config/typescript.json

Original file line number	Diff line number	Diff line change
`@@ -7,5 +7,6 @@`
`7`	`7`	`"coverageReporters": ["cobertura", "html"],`
`8`	`8`
`9`	`9`	`// Use v8 coverage provider to avoid Babel`
`10`		`- "coverageProvider": "v8"`
	`10`	`+ "coverageProvider": "v8",`
	`11`	`+ "resolver": "@rushstack/heft-jest-plugin/lib/exports/jest-node-modules-symlink-resolver"`
`11`	`12`	`}`
Original file line number	Diff line number	Diff line change
`@@ -49,5 +49,7 @@`
`49`	`49`	`// "excludeGlobs": [`
`50`	`50`	`// "some/path/*.css"`
`51`	`51`	`// ]`
`52`		`- }`
	`52`	`+ },`
	`53`	`+`
	`54`	`+ "onlyResolveSymlinksInNodeModules": true`
`53`	`55`	`}`