Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

perf(cli): doc types and props checks reuse parsed files (TS 5.3+) (#9517)

Christopher Chedeau4931743e460b
diff --git a/.changeset/cli-source-files-kept.md b/.changeset/cli-source-files-kept.md
new file mode 100644
index 0000000000..2e4cc94474
--- /dev/null
+++ b/.changeset/cli-source-files-kept.md
@@ -0,0 +1,7 @@
+---
+'@astryxdesign/cli': patch
+---
+
+[perf] With TypeScript 5.3 or later, `validateIntegration()`, `integrationComponentConflicts()` and `integrationCheck()` from `@astryxdesign/cli/api`, and the commands that run them, reuse each TypeScript file a check parsed for later checks in the same process that parse it the same way, while the file is unchanged, instead of parsing it again for every check. Their results are unchanged.
+
+@vjeux
diff --git a/packages/cli/foundation/discovery/doc-types.mjs b/packages/cli/foundation/discovery/doc-types.mjs
index 74ac4c712e..e880520769 100644
--- a/packages/cli/foundation/discovery/doc-types.mjs
+++ b/packages/cli/foundation/discovery/doc-types.mjs
@@ -45,6 +45,7 @@ import {
TYPESCRIPT_FIX,
unusableTypeScriptReason,
} from './project-typescript.mjs';
+import {keepSourceFiles} from './source-file-cache.mjs';
/**
* @typedef {'components' | 'hooks' | 'templates' | 'docs' | 'docs-overlay' | 'themes' | 'icon-sets'} DocRoot
@@ -1035,8 +1036,9 @@ export async function checkDocTypes(files, options) {
paths: {[AUTHORING]: [AUTHORING_TYPES]},
};
const host = ts.createCompilerHost(options_);
- const getSourceFile = host.getSourceFile.bind(host);
- // Every other file is parsed once, so a later pass reuses it.
+ // Every other file is parsed once, so a later pass reuses it; and once per
+ // process for these options, so a later check does too.
+ const getSourceFile = keepSourceFiles(ts, host, options_);
/** @type {Map<string, any>} */
const parsed = new Map();
host.getSourceFile = (
diff --git a/packages/cli/foundation/discovery/prop-drift.mjs b/packages/cli/foundation/discovery/prop-drift.mjs
index 2be419511a..e92e690134 100644
--- a/packages/cli/foundation/discovery/prop-drift.mjs
+++ b/packages/cli/foundation/discovery/prop-drift.mjs
@@ -54,6 +54,7 @@ import {
TYPESCRIPT_FIX,
unusableTypeScriptReason,
} from './project-typescript.mjs';
+import {keepSourceFiles} from './source-file-cache.mjs';
import {judgeType} from './type-match.mjs';
import {typeTextTooOld} from './type-text.mjs';
import {
@@ -1802,6 +1803,10 @@ export function comparePropTables(tables, options) {
typesNotCompared: [],
};
}
+ // Parsed once per process for these options: every check reads the same
+ // library and package declarations.
+ const host = ts.createCompilerHost(project.options);
+ host.getSourceFile = keepSourceFiles(ts, host, project.options);
const program = ts.createProgram({
rootNames: [
...new Set([
@@ -1810,6 +1815,7 @@ export function comparePropTables(tables, options) {
]),
],
options: project.options,
+ host,
});
const checker = program.getTypeChecker();
const isOwnFile = ownFileTest(root);
diff --git a/packages/cli/foundation/discovery/source-file-cache.mjs b/packages/cli/foundation/discovery/source-file-cache.mjs
new file mode 100644
index 0000000000..3ed2b1592b
--- /dev/null
+++ b/packages/cli/foundation/discovery/source-file-cache.mjs
@@ -0,0 +1,140 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file source-file-cache.mjs
+ * @input A TypeScript compiler host, the project's own `typescript`, and the
+ * compiler options its programs use
+ * @output `keepSourceFiles`: the host's `getSourceFile`, keeping each parsed
+ * file while its stamp (`fileStamp`) is the same, for every program in this
+ * process whose options parse and bind a file alike.
+ * @position The props comparison (./prop-drift.mjs) and the doc types check
+ * (./doc-types.mjs) build a program for every check, and each parsed the
+ * same declarations again: the TypeScript library's, React's, the
+ * authoring types. A process that checks more than once (a test run, a
+ * long-running caller, a check over several roots) now parses each once.
+ * TypeScript's language service shares parsed files between programs the
+ * same way; from 5.3 a program keeps what it resolved for a file itself, not
+ * on the file, which is what makes sharing them safe across programs.
+ */
+
+import path from 'node:path';
+import {fileStamp, recordContentRead} from '../fs/read-record.mjs';
+
+/**
+ * The files kept, per TypeScript and per how the options parse and bind a
+ * file (`settingsKey`), the most recently used options last.
+ * @type {WeakMap<object, Map<string, Map<string, {stamp: string, sourceFile: any}>>>}
+ */
+const kept = new WeakMap();
+
+/** The most files kept for one key; the oldest go first. */
+const MAX_FILES = 5000;
+
+/**
+ * The most keys kept per TypeScript; the least recently used go first. Each
+ * holds its own parse of the TypeScript library's declarations, the largest
+ * files a program reads.
+ */
+const MAX_KEYS = 8;
+
+/** @type {WeakMap<object, any>} one document registry per TypeScript */
+const registries = new WeakMap();
+
+/**
+ * What in `options` decides how a file parses and binds: the key TypeScript's
+ * language service shares parsed files by (its document registry's), and
+ * whether a script binds in strict mode, which that key reads only from an
+ * explicit `alwaysStrict`. Options that only resolve or check (`paths`,
+ * `types`, `lib`, the strict checks) do not change it.
+ * @param {any} ts
+ * @param {any} options
+ * @returns {string}
+ */
+function settingsKey(ts, options) {
+ let registry = registries.get(ts);
+ if (registry == null) {
+ registry = ts.createDocumentRegistry();
+ registries.set(ts, registry);
+ }
+ const strict =
+ typeof ts.getStrictOptionValue === 'function'
+ ? ts.getStrictOptionValue(options, 'alwaysStrict')
+ : (options.alwaysStrict ?? options.strict ?? false);
+ return `${registry.getKeyForCompilationSettings(options)}|${strict ? 1 : 0}`;
+}
+
+/**
+ * Whether this TypeScript keeps a program's module resolutions on the program
+ * (5.3 and later) rather than on the parsed files it shares.
+ * @param {any} ts
+ */
+function sharesSafely(ts) {
+ const [major, minor] = String(ts.versionMajorMinor ?? '0.0')
+ .split('.')
+ .map(Number);
+ return major > 5 || (major === 5 && minor >= 3);
+}
+
+/**
+ * `host.getSourceFile`, keeping each file it parses while the file's stamp is
+ * the same, for every program whose options parse and bind a file as
+ * `options` do. A kept file still counts as read by the running operation
+ * (`recordContentRead`), as a parse would. On a TypeScript older than 5.3 the
+ * host's own `getSourceFile` is returned.
+ * @param {any} ts the project's own `typescript`
+ * @param {any} host
+ * @param {any} options the compiler options of the programs that use it
+ * @returns {(fileName: string, languageVersionOrOptions: any, onError?: any, shouldCreate?: boolean) => any}
+ */
+export function keepSourceFiles(ts, host, options) {
+ const getSourceFile = host.getSourceFile.bind(host);
+ if (!sharesSafely(ts)) return getSourceFile;
+ let bySettings = kept.get(ts);
+ if (bySettings == null) {
+ bySettings = new Map();
+ kept.set(ts, bySettings);
+ }
+ const settings = settingsKey(ts, options);
+ const byFile = bySettings.get(settings) ?? new Map();
+ // Most recently used last; past MAX_KEYS, the least recently used go.
+ bySettings.delete(settings);
+ bySettings.set(settings, byFile);
+ if (bySettings.size > MAX_KEYS) {
+ bySettings.delete(/** @type {string} */ (bySettings.keys().next().value));
+ }
+ return (fileName, languageVersionOrOptions, onError, shouldCreate) => {
+ // How the file parses: its language version and, from TypeScript 5.0,
+ // its module format and JSDoc parsing (the module detection the options
+ // give is in `settings`).
+ const parse =
+ languageVersionOrOptions != null &&
+ typeof languageVersionOrOptions === 'object'
+ ? [
+ languageVersionOrOptions.languageVersion,
+ languageVersionOrOptions.impliedNodeFormat,
+ languageVersionOrOptions.jsDocParsingMode,
+ ]
+ : [languageVersionOrOptions];
+ const key = `${path.resolve(fileName)}\0${JSON.stringify(parse)}`;
+ const stamp = fileStamp(fileName);
+ const seen = byFile.get(key);
+ if (seen != null && stamp !== '-' && seen.stamp === stamp) {
+ recordContentRead(fileName);
+ return seen.sourceFile;
+ }
+ const sourceFile = getSourceFile(
+ fileName,
+ languageVersionOrOptions,
+ onError,
+ shouldCreate,
+ );
+ byFile.delete(key);
+ if (sourceFile != null && stamp !== '-') {
+ byFile.set(key, {stamp, sourceFile});
+ if (byFile.size > MAX_FILES) {
+ byFile.delete(/** @type {string} */ (byFile.keys().next().value));
+ }
+ }
+ return sourceFile;
+ };
+}
diff --git a/packages/cli/foundation/discovery/source-file-cache.test.mjs b/packages/cli/foundation/discovery/source-file-cache.test.mjs
new file mode 100644
index 0000000000..b899b89c4a
--- /dev/null
+++ b/packages/cli/foundation/discovery/source-file-cache.test.mjs
@@ -0,0 +1,165 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file source-file-cache.test.mjs
+ * @input source-file-cache.mjs's `keepSourceFiles`, with the repository's own
+ * TypeScript and a file in a temporary folder
+ * @output Unit cases: a file is parsed once for every host and program that
+ * compiles with the same options while it is unchanged; a rewrite at the
+ * same size and modification time is parsed again; options that only
+ * resolve or check share files, and options that parse or bind them
+ * differently (JSX, strict mode) keep them apart; a kept file counts as read by the running operation; a
+ * TypeScript older than 5.3 gets the host's own `getSourceFile`; and a
+ * program over kept files reads the same types as one over fresh parses.
+ * @position Node unit tests beside the props comparison and doc types check
+ * that use it.
+ *
+ * SYNC: When modified, update source-file-cache.mjs.
+ */
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import ts from 'typescript';
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+import {recordReads} from '../fs/read-record.mjs';
+import {keepSourceFiles} from './source-file-cache.mjs';
+
+const OPTIONS = {
+ target: ts.ScriptTarget.ES2022,
+ module: ts.ModuleKind.ESNext,
+ moduleResolution: ts.ModuleResolutionKind.Bundler,
+ strict: true,
+ noEmit: true,
+ types: [],
+};
+
+let dir;
+let file;
+
+beforeEach(() => {
+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-source-file-cache-'));
+ file = path.join(dir, 'props.ts');
+ fs.writeFileSync(file, 'export interface Props {size: number}\n');
+});
+
+afterEach(() => {
+ fs.rmSync(dir, {recursive: true, force: true});
+});
+
+/** A host for `options` whose `getSourceFile` counts its parses of `file`. */
+function countingHost(options) {
+ const host = ts.createCompilerHost(options);
+ const parse = host.getSourceFile.bind(host);
+ const counted = {host, parses: 0};
+ host.getSourceFile = (name, ...rest) => {
+ if (path.resolve(name) === path.resolve(file)) counted.parses += 1;
+ return parse(name, ...rest);
+ };
+ return counted;
+}
+
+describe('keepSourceFiles', () => {
+ it('parses a file once for every host that compiles with the same options, while it is unchanged', () => {
+ const first = countingHost(OPTIONS);
+ const second = countingHost({...OPTIONS});
+ const fromFirst = keepSourceFiles(ts, first.host, OPTIONS);
+ const fromSecond = keepSourceFiles(ts, second.host, {...OPTIONS});
+ const parsed = fromFirst(file, ts.ScriptTarget.ES2022);
+ expect(fromFirst(file, ts.ScriptTarget.ES2022)).toBe(parsed);
+ expect(fromSecond(file, ts.ScriptTarget.ES2022)).toBe(parsed);
+ expect(first.parses + second.parses).toBe(1);
+ });
+
+ it('parses it again once it is rewritten, at the same size and modification time', () => {
+ const {host} = countingHost(OPTIONS);
+ const keep = keepSourceFiles(ts, host, OPTIONS);
+ const before = keep(file, ts.ScriptTarget.ES2022);
+ const {atime, mtime} = fs.statSync(file);
+ fs.writeFileSync(file, 'export interface Props {name: string}\n');
+ fs.utimesSync(file, atime, mtime);
+ const after = keep(file, ts.ScriptTarget.ES2022);
+ expect(after).not.toBe(before);
+ expect(after.text).toContain('name: string');
+ });
+
+ it('shares files between options that parse and bind alike', () => {
+ const {host} = countingHost(OPTIONS);
+ const resolving = {...OPTIONS, types: ['node'], paths: {'@x/*': ['./x/*']}};
+ const parsed = keepSourceFiles(
+ ts,
+ host,
+ OPTIONS,
+ )(file, ts.ScriptTarget.ES2022);
+ const other = countingHost(resolving);
+ expect(
+ keepSourceFiles(ts, other.host, resolving)(file, ts.ScriptTarget.ES2022),
+ ).toBe(parsed);
+ expect(other.parses).toBe(0);
+ });
+
+ it('keeps files apart for options that parse or bind them differently', () => {
+ const {host} = countingHost(OPTIONS);
+ const parsed = keepSourceFiles(
+ ts,
+ host,
+ OPTIONS,
+ )(file, ts.ScriptTarget.ES2022);
+ for (const options of [
+ {...OPTIONS, jsx: ts.JsxEmit.ReactJSX},
+ {...OPTIONS, strict: false},
+ ]) {
+ expect(
+ keepSourceFiles(
+ ts,
+ countingHost(options).host,
+ options,
+ )(file, ts.ScriptTarget.ES2022),
+ ).not.toBe(parsed);
+ }
+ });
+
+ it('counts a kept file as read by the running operation', () => {
+ const {host} = countingHost(OPTIONS);
+ const keep = keepSourceFiles(ts, host, OPTIONS);
+ keep(file, ts.ScriptTarget.ES2022);
+ const seen = new Map();
+ recordReads(seen, () => keep(file, ts.ScriptTarget.ES2022));
+ expect([...seen.keys()]).toContain(path.resolve(file));
+ });
+
+ it("gives a TypeScript older than 5.3 the host's own getSourceFile", () => {
+ const counted = countingHost(OPTIONS);
+ const older = {...ts, versionMajorMinor: '5.2'};
+ const keep = keepSourceFiles(older, counted.host, OPTIONS);
+ keep(file, ts.ScriptTarget.ES2022);
+ keep(file, ts.ScriptTarget.ES2022);
+ expect(counted.parses).toBe(2);
+ });
+
+ it('reads the same types in a program over kept files as over fresh parses', () => {
+ const propsOf = keep => {
+ const host = ts.createCompilerHost(OPTIONS);
+ if (keep) host.getSourceFile = keepSourceFiles(ts, host, OPTIONS);
+ const program = ts.createProgram({
+ rootNames: [file],
+ options: OPTIONS,
+ host,
+ });
+ const checker = program.getTypeChecker();
+ const source = program.getSourceFile(file);
+ const props = checker
+ .getExportsOfModule(checker.getSymbolAtLocation(source))
+ .find(symbol => symbol.name === 'Props');
+ return checker.typeToString(
+ checker.getDeclaredTypeOfSymbol(props),
+ undefined,
+ ts.TypeFormatFlags.InTypeAlias,
+ );
+ };
+ const fresh = propsOf(false);
+ // Twice over kept files: the second program binds nothing anew.
+ expect(propsOf(true)).toBe(fresh);
+ expect(propsOf(true)).toBe(fresh);
+ });
+});