An open source design system that's fully customizable and agent ready
TypeScript
perf(cli): read each source directory once per search or build (#7421)
Cindy Zhang9cbecf2691d7
diff --git a/.changeset/cli-search-build-read-tree-once.md b/.changeset/cli-search-build-read-tree-once.mdnew file mode 100644index 0000000000..b181023a77--- /dev/null+++ b/.changeset/cli-search-build-read-tree-once.md@@ -0,0 +1,7 @@+---+'@astryxdesign/cli': patch+---++[perf] `astryx search` and `astryx build`, and the `search` and `build` API functions, read each source directory once per call instead of once per candidate. Results are unchanged. A warm `search` drops from about 1.9 s to 0.4 s on macOS.++@cixzhangdiff --git a/packages/cli/api/build/_adapter.mjs b/packages/cli/api/build/_adapter.mjsindex 5cc4053f68..c1b03e20f1 100644--- a/packages/cli/api/build/_adapter.mjs+++ b/packages/cli/api/build/_adapter.mjs@@ -23,6 +23,13 @@ import {componentKeywords} from '../search/search.mjs';import {findCoreDir} from '../../foundation/fs/paths.mjs';import {analyzeTemplateNeeds} from '../../foundation/discovery/template-needs.mjs';+/**+ * Runs one build with a single directory listing per path: the kit resolves+ * every candidate's source and docs by walking the same tree, which does not+ * change during the call.+ */+export {withListingCache} from '../../foundation/fs/listing-cache.mjs';+/*** A page template the kit can recommend starting from.* @typedef {object} PageTemplatediff --git a/packages/cli/api/build/build.mjs b/packages/cli/api/build/build.mjsindex e1c651bea6..c77028c955 100644--- a/packages/cli/api/build/build.mjs+++ b/packages/cli/api/build/build.mjs@@ -17,6 +17,7 @@ import {AstryxError} from '../error.mjs';import {ERROR_CODES} from '../../foundation/response/error-codes.mjs';import {buildHelp} from './help/help.mjs';import {buildKit} from './kit/kit.mjs';+import {withListingCache} from './_adapter.mjs';// Barrel: surface the leaves alongside the dispatcher for the fractal shape.export {buildHelp, buildKit};@@ -40,5 +41,7 @@ export async function build(query, options = {}) {ERROR_CODES.ERR_INVALID_ARGUMENT,);}- return buildKit(query, options);+ // The kit resolves every candidate's source and docs by walking the same+ // tree; read each directory once for this call.+ return withListingCache(() => buildKit(query, options));}diff --git a/packages/cli/api/search/search.mjs b/packages/cli/api/search/search.mjsindex a6077e5d7a..2f12d86f33 100644--- a/packages/cli/api/search/search.mjs+++ b/packages/cli/api/search/search.mjs@@ -79,6 +79,7 @@* queries they have nothing to do with.*/+import {withListingCache} from '../../foundation/fs/listing-cache.mjs';import {readDocView} from '../../foundation/doc-compiler/read.mjs';import {findCoreDir} from '../../foundation/fs/paths.mjs';import {@@ -1769,6 +1770,17 @@ function toResult(c, score, reason, matchedTerms, queryTerms) {* @returns {Promise<import('./search.type.mjs').SearchResponse>}*/export async function search(query, options = {}) {+ // Every candidate resolves its source and docs by walking the same tree;+ // read each directory once for this call.+ return withListingCache(() => searchOnce(query, options));+}++/**+ * @param {string} query+ * @param {NonNullable<Parameters<typeof search>[1]>} options+ * @returns {ReturnType<typeof search>}+ */+async function searchOnce(query, options) {const {cwd = process.cwd(), type, limit = 20} = options;if (!query || !String(query).trim()) {diff --git a/packages/cli/foundation/discovery/component-discovery.mjs b/packages/cli/foundation/discovery/component-discovery.mjsindex fd8237e69f..53802c50b7 100644--- a/packages/cli/foundation/discovery/component-discovery.mjs+++ b/packages/cli/foundation/discovery/component-discovery.mjs@@ -6,6 +6,7 @@import * as fs from 'node:fs';import * as path from 'node:path';+import {listDir, listDirEntries, pathExists} from '../fs/listing-cache.mjs';import {existsCaseExact} from '../fs/paths.mjs';import {CORE_PROVIDER_ID} from '../identity/providers.mjs';import {loadComponentDoc} from './component-loader.mjs';@@ -52,8 +53,8 @@ function isComponentSourceFile(fileName, dirPath) {if (BARE_COMPONENT_FILE_RE.test(fileName)) {const base = fileName.replace(/\.tsx$/, '');return (- fs.existsSync(path.join(dirPath, `${base}.doc.mjs`)) ||- fs.existsSync(path.join(dirPath, `XDS${base}.doc.mjs`))+ pathExists(path.join(dirPath, `${base}.doc.mjs`)) ||+ pathExists(path.join(dirPath, `XDS${base}.doc.mjs`)));}return false;@@ -132,8 +133,8 @@ export function discoverComponents(coreDir) {function collectXDSFiles(dirPath) {/** @type {string[]} */const results = [];- if (!fs.existsSync(dirPath)) return results;- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ if (!pathExists(dirPath)) return results;+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.isDirectory()) {results.push(...collectXDSFiles(path.join(dirPath, entry.name)));@@ -144,7 +145,7 @@ export function discoverComponents(coreDir) {return results;}- const topEntries = fs.readdirSync(srcDir, {withFileTypes: true});+ const topEntries = listDirEntries(srcDir);for (const entry of topEntries) {if (!entry.isDirectory() || SKIP_DIRS.has(entry.name)) continue;@@ -154,10 +155,10 @@ export function discoverComponents(coreDir) {// Read the group from the directory's .doc.mjs file (if it exists)// Check both {Name}.doc.mjs and Astryx{Name}.doc.mjs naming conventionslet docFile = path.join(dirPath, `${entry.name}.doc.mjs`);- if (!fs.existsSync(docFile)) {+ if (!pathExists(docFile)) {docFile = path.join(dirPath, `XDS${entry.name}.doc.mjs`);}- const {group, hiddenComponents, hidden} = fs.existsSync(docFile)+ const {group, hiddenComponents, hidden} = pathExists(docFile)? readDocMeta(docFile): {group: null, hiddenComponents: new Set(), hidden: false};@@ -171,10 +172,10 @@ export function discoverComponents(coreDir) {// Check for a per-component doc file that overrides the directory grouplet compGroup = group;let compDoc = path.join(dirPath, `${componentName}.doc.mjs`);- if (!fs.existsSync(compDoc)) {+ if (!pathExists(compDoc)) {compDoc = path.join(dirPath, `XDS${componentName}.doc.mjs`);}- const hasComponentDoc = fs.existsSync(compDoc);+ const hasComponentDoc = pathExists(compDoc);if (hasComponentDoc) {const compMeta = readDocMeta(compDoc);if (compMeta.group) compGroup = compMeta.group;@@ -183,7 +184,7 @@ export function discoverComponents(coreDir) {// Skip components without any .doc.mjs file (directory-level or// component-level). They can't be documented, so surfacing them in// the component list leads to broken docs-site links.- if (!fs.existsSync(docFile) && !hasComponentDoc) continue;+ if (!pathExists(docFile) && !hasComponentDoc) continue;if (!componentGroups.has(componentName)) {componentGroups.set(componentName, compGroup);@@ -251,7 +252,7 @@ export function findComponentReadme(coreDir, name) {if (existsCaseExact(directXds, srcDir)) return directXds;// Nested match: src/*/{name}/{Name}.doc.mjs or src/*/{name}/Astryx{Name}.doc.mjs- const entries = fs.readdirSync(srcDir, {withFileTypes: true});+ const entries = listDirEntries(srcDir);for (const entry of entries) {if (!entry.isDirectory()) continue;const nested = path.join(srcDir, entry.name, name, exactDoc);@@ -275,7 +276,7 @@ export function findComponentReadme(coreDir, name) {if (sourcePath) {let dir = path.dirname(sourcePath);while (dir.startsWith(srcDir)) {- const dirEntries = fs.readdirSync(dir);+ const dirEntries = listDir(dir);for (const f of dirEntries) {if (f.endsWith('.doc.mjs')) return path.join(dir, f);}@@ -318,8 +319,8 @@ export function findComponentSource(coreDir, name) {* @returns {string | null}*/function searchDir(dirPath) {- if (!fs.existsSync(dirPath)) return null;- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ if (!pathExists(dirPath)) return null;+ const entries = listDirEntries(dirPath);// Check for an exact match (prefixed or bare) firstfor (const candidate of candidateFiles) {@@ -345,7 +346,7 @@ export function findComponentSource(coreDir, name) {}// Search nested (component might be under a parent dir)- const entries = fs.readdirSync(srcDir, {withFileTypes: true});+ const entries = listDirEntries(srcDir);for (const entry of entries) {if (!entry.isDirectory()) continue;const nestedDir = path.join(srcDir, entry.name, name);@@ -369,7 +370,7 @@ export function findComponentSource(coreDir, name) {export function resolveImportPath(coreDir, componentName) {const srcDir = path.join(coreDir, 'src');const pkgPath = path.join(coreDir, 'package.json');- const pkg = fs.existsSync(pkgPath)+ const pkg = pathExists(pkgPath)? JSON.parse(fs.readFileSync(pkgPath, 'utf-8')): null;@@ -522,14 +523,14 @@ function readPackageExports(packageDir) {* @returns {Record<string, string[]>}*/export function discoverExternalComponentsGrouped(docsDir) {- if (!fs.existsSync(docsDir)) return {};+ if (!pathExists(docsDir)) return {};/** @type {Map<string, string|null>} componentName → group */const componentGroups = new Map();/** @param {string} dirPath */function scanDir(dirPath) {- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.name === 'node_modules' || entry.name === '__tests__') continue;const fullPath = path.join(dirPath, entry.name);@@ -593,7 +594,7 @@ export function discoverExternalComponentsGrouped(docsDir) {* @returns {string | null}*/export function findExternalComponentDoc(docsDir, name) {- if (!fs.existsSync(docsDir)) return null;+ if (!pathExists(docsDir)) return null;const target = `${name}.doc.mjs`;/**@@ -601,7 +602,7 @@ export function findExternalComponentDoc(docsDir, name) {* @returns {string | null}*/function scanDir(dirPath) {- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.name === 'node_modules' || entry.name === '__tests__') continue;const fullPath = path.join(dirPath, entry.name);@@ -642,7 +643,7 @@ function integrationSourceForDoc(docPath) {const dir = path.dirname(docPath);const base = path.basename(docPath).replace(/\.doc\.(ts|mjs|js)$/, '');const candidate = path.join(dir, `${base}.tsx`);- return fs.existsSync(candidate) ? candidate : null;+ return pathExists(candidate) ? candidate : null;}/**@@ -656,14 +657,14 @@ function integrationSourceForDoc(docPath) {*/export function discoverIntegrationComponents(integration) {const componentsDir = integration?.components;- if (!componentsDir || !fs.existsSync(componentsDir)) return [];+ if (!componentsDir || !pathExists(componentsDir)) return [];/** @type {Map<string, {name: string, package: string, docPath: string, sourcePath: string|null, issuesUrl: string|undefined, group: string|null}>} */const byName = new Map();/** @param {string} dirPath */function scanDir(dirPath) {- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.name === 'node_modules' || entry.name === '__tests__') continue;const fullPath = path.join(dirPath, entry.name);@@ -757,18 +758,18 @@ export async function discoverValidIntegrationComponents(integration) {*/export function findIntegrationComponentDoc(integration, name) {const componentsDir = integration?.components;- if (!componentsDir || !fs.existsSync(componentsDir)) return null;+ if (!componentsDir || !pathExists(componentsDir)) return null;/*** @param {string} dirPath* @returns {string | null}*/function scanDir(dirPath) {- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ const entries = listDirEntries(dirPath);// Exact same-stem match (precedence order) first in this dir.for (const suffix of INTEGRATION_DOC_SUFFIXES) {const candidate = path.join(dirPath, `${name}${suffix}`);- if (fs.existsSync(candidate)) return candidate;+ if (pathExists(candidate)) return candidate;}for (const entry of entries) {if (entry.name === 'node_modules' || entry.name === '__tests__') continue;diff --git a/packages/cli/foundation/discovery/hook-discovery.mjs b/packages/cli/foundation/discovery/hook-discovery.mjsindex e3f02dc13e..93d8e29a26 100644--- a/packages/cli/foundation/discovery/hook-discovery.mjs+++ b/packages/cli/foundation/discovery/hook-discovery.mjs@@ -6,6 +6,7 @@import * as fs from 'node:fs';import * as path from 'node:path';+import {listDir, listDirEntries, pathExists} from '../fs/listing-cache.mjs';import {existsCaseExact} from '../fs/paths.mjs';import {levenshteinDistance} from '../text/string-utils.mjs';@@ -57,8 +58,8 @@ export function discoverHooks(coreDir) {const hookCategories = new Map();// 1. Scan src/hooks/ for *.doc.mjs files- if (fs.existsSync(hooksDir)) {- const entries = fs.readdirSync(hooksDir, {withFileTypes: true});+ if (pathExists(hooksDir)) {+ const entries = listDirEntries(hooksDir);for (const entry of entries) {if (!entry.isFile()) continue;if (!entry.name.endsWith('.doc.mjs')) continue;@@ -70,8 +71,8 @@ export function discoverHooks(coreDir) {}// 2. Scan ALL component directories for files matching use*.doc.mjs- if (fs.existsSync(srcDir)) {- const topEntries = fs.readdirSync(srcDir, {withFileTypes: true});+ if (pathExists(srcDir)) {+ const topEntries = listDirEntries(srcDir);for (const entry of topEntries) {if (!entry.isDirectory() || SKIP_DIRS.has(entry.name) || entry.name === 'hooks') continue;const dirPath = path.join(srcDir, entry.name);@@ -114,8 +115,8 @@ export function discoverHooks(coreDir) {* @param {Map<string, string>} hookCategories*/function scanDirForHookDocs(dirPath, hookCategories) {- if (!fs.existsSync(dirPath)) return;- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ if (!pathExists(dirPath)) return;+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {scanDirForHookDocs(path.join(dirPath, entry.name), hookCategories);@@ -162,8 +163,8 @@ export function findHookDoc(coreDir, name) {}// 2. Case-insensitive search in hooks directory- if (fs.existsSync(hooksDir)) {- const entries = fs.readdirSync(hooksDir);+ if (pathExists(hooksDir)) {+ const entries = listDir(hooksDir);for (const entry of entries) {if (!entry.endsWith('.doc.mjs')) continue;const hookName = hookNameFromFile(entry);@@ -175,8 +176,8 @@ export function findHookDoc(coreDir, name) {}// 3. Check all component dirs for {name}.doc.mjs or use{Name}.doc.mjs- if (fs.existsSync(srcDir)) {- const topEntries = fs.readdirSync(srcDir, {withFileTypes: true});+ if (pathExists(srcDir)) {+ const topEntries = listDirEntries(srcDir);for (const entry of topEntries) {if (!entry.isDirectory() || SKIP_DIRS.has(entry.name) || entry.name === 'hooks') continue;const found = searchDirForHookDoc(path.join(srcDir, entry.name), uniqueCandidates);@@ -213,8 +214,8 @@ export function findHookDoc(coreDir, name) {* @returns {string | null}*/function searchDirForHookDoc(dirPath, candidates) {- if (!fs.existsSync(dirPath)) return null;- const entries = fs.readdirSync(dirPath, {withFileTypes: true});+ if (!pathExists(dirPath)) return null;+ const entries = listDirEntries(dirPath);for (const entry of entries) {if (entry.isDirectory() && !SKIP_DIRS.has(entry.name)) {const found = searchDirForHookDoc(path.join(dirPath, entry.name), candidates);diff --git a/packages/cli/foundation/fs/listing-cache.mjs b/packages/cli/foundation/fs/listing-cache.mjsnew file mode 100644index 0000000000..bfb2237013--- /dev/null+++ b/packages/cli/foundation/fs/listing-cache.mjs@@ -0,0 +1,95 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file One directory listing per path for the duration of one read-only+ * operation.+ *+ * @input A read-only operation (`withListingCache(fn)`), and directory paths+ * its discovery helpers list or probe.+ * @output The same answers `fs.readdirSync` / `fs.existsSync` give, read from+ * disk once per path while the operation runs.+ * @position Used by the name resolvers in foundation/discovery and by+ * `existsCaseExact`. A search or build resolves ~160 components and their+ * hooks, and each resolution walks the same source tree again; inside one+ * operation the tree does not change, so each directory is read once.+ * Outside `withListingCache` every helper reads the disk directly, exactly+ * as before, so long-lived callers never see a stale listing.+ */++import {AsyncLocalStorage} from 'node:async_hooks';+import * as fs from 'node:fs';++/**+ * @typedef {{+ * names: Map<string, string[]>,+ * entries: Map<string, fs.Dirent[]>,+ * exists: Map<string, boolean>,+ * }} ListingCache+ */++/** @type {AsyncLocalStorage<ListingCache>} */+const scope = new AsyncLocalStorage();++/**+ * Run `fn` with a listing cache. A nested call shares the outer cache. Only+ * wrap operations that do not change the files they list.+ * @template T+ * @param {() => T} fn+ * @returns {T}+ */+export function withListingCache(fn) {+ if (scope.getStore()) return fn();+ return scope.run(+ {names: new Map(), entries: new Map(), exists: new Map()},+ fn,+ );+}++/**+ * `fs.readdirSync(dir)`, read once per operation. Returns a fresh array, so a+ * caller that sorts it cannot reorder another caller's listing.+ * @param {string} dir+ * @returns {string[]}+ */+export function listDir(dir) {+ const cache = scope.getStore();+ if (!cache) return fs.readdirSync(dir);+ let names = cache.names.get(dir);+ if (!names) {+ names = fs.readdirSync(dir);+ cache.names.set(dir, names);+ }+ return names.slice();+}++/**+ * `fs.readdirSync(dir, {withFileTypes: true})`, read once per operation.+ * @param {string} dir+ * @returns {fs.Dirent[]}+ */+export function listDirEntries(dir) {+ const cache = scope.getStore();+ if (!cache) return fs.readdirSync(dir, {withFileTypes: true});+ let entries = cache.entries.get(dir);+ if (!entries) {+ entries = fs.readdirSync(dir, {withFileTypes: true});+ cache.entries.set(dir, entries);+ }+ return entries.slice();+}++/**+ * `fs.existsSync(target)`, probed once per operation.+ * @param {string} target+ * @returns {boolean}+ */+export function pathExists(target) {+ const cache = scope.getStore();+ if (!cache) return fs.existsSync(target);+ let exists = cache.exists.get(target);+ if (exists === undefined) {+ exists = fs.existsSync(target);+ cache.exists.set(target, exists);+ }+ return exists;+}diff --git a/packages/cli/foundation/fs/listing-cache.test.mjs b/packages/cli/foundation/fs/listing-cache.test.mjsnew file mode 100644index 0000000000..9696104ca5--- /dev/null+++ b/packages/cli/foundation/fs/listing-cache.test.mjs@@ -0,0 +1,88 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {afterEach, beforeEach, describe, expect, it} from 'vitest';+import * as fs from 'node:fs';+import * as os from 'node:os';+import * as path from 'node:path';+import {+ listDir,+ listDirEntries,+ pathExists,+ withListingCache,+} from './listing-cache.mjs';+import {existsCaseExact} from './paths.mjs';++let dir;++beforeEach(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-listing-cache-'));+ fs.writeFileSync(path.join(dir, 'b.txt'), '');+ fs.writeFileSync(path.join(dir, 'a.txt'), '');+});++afterEach(() => {+ fs.rmSync(dir, {recursive: true, force: true});+});++describe('listing cache', () => {+ it('reads the disk directly outside an operation', () => {+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt']);+ fs.writeFileSync(path.join(dir, 'c.txt'), '');+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt', 'c.txt']);+ expect(pathExists(path.join(dir, 'c.txt'))).toBe(true);+ });++ it('answers from one read for the rest of an operation', async () => {+ await withListingCache(async () => {+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt']);+ expect(pathExists(path.join(dir, 'c.txt'))).toBe(false);+ fs.writeFileSync(path.join(dir, 'c.txt'), '');+ await new Promise(resolve => setTimeout(resolve, 0));+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt']);+ expect(pathExists(path.join(dir, 'c.txt'))).toBe(false);+ expect(+ listDirEntries(dir)+ .map(entry => entry.name)+ .sort(),+ ).toEqual(['a.txt', 'b.txt', 'c.txt']);+ });+ // The operation's snapshot does not outlive it.+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt', 'c.txt']);+ });++ it('shares one cache with nested operations', () => {+ withListingCache(() => {+ listDir(dir);+ fs.writeFileSync(path.join(dir, 'c.txt'), '');+ withListingCache(() => {+ expect(listDir(dir).sort()).toEqual(['a.txt', 'b.txt']);+ });+ });+ });++ it('hands every caller its own array', () => {+ withListingCache(() => {+ listDir(dir).reverse().push('mutated');+ expect(listDir(dir)).not.toContain('mutated');+ listDirEntries(dir).length = 0;+ expect(listDirEntries(dir)).toHaveLength(2);+ });+ });++ it('does not remember a failed read', () => {+ const missing = path.join(dir, 'later');+ withListingCache(() => {+ expect(() => listDir(missing)).toThrow();+ fs.mkdirSync(missing);+ expect(listDir(missing)).toEqual([]);+ });+ });++ it('keeps existsCaseExact exact-case inside an operation', () => {+ fs.mkdirSync(path.join(dir, 'Button'));+ withListingCache(() => {+ expect(existsCaseExact(path.join(dir, 'Button'), dir)).toBe(true);+ expect(existsCaseExact(path.join(dir, 'button'), dir)).toBe(false);+ });+ });+});diff --git a/packages/cli/foundation/fs/paths.mjs b/packages/cli/foundation/fs/paths.mjsindex d6e3847568..1232336830 100644--- a/packages/cli/foundation/fs/paths.mjs+++ b/packages/cli/foundation/fs/paths.mjs@@ -10,6 +10,7 @@ import * as fs from 'node:fs';import {createRequire} from 'node:module';import * as path from 'node:path';import {fileURLToPath} from 'node:url';+import {listDir, pathExists} from './listing-cache.mjs';const __dirname = path.dirname(fileURLToPath(import.meta.url));@@ -231,7 +232,7 @@ export function listComponents(coreDir) {* @returns {boolean}*/export function existsCaseExact(targetPath, rootDir) {- if (!fs.existsSync(targetPath)) return false;+ if (!pathExists(targetPath)) return false;const rel = path.relative(rootDir, targetPath);if (rel === '') return true;@@ -241,7 +242,7 @@ export function existsCaseExact(targetPath, rootDir) {for (const segment of rel.split(path.sep)) {let entries;try {- entries = fs.readdirSync(dir);+ entries = listDir(dir);} catch {return false;}