Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

perf(cli): search parses a page template only when a term can match it (#9871)

Christopher Chedeau442f2ddd4f05
diff --git a/.changeset/cli-search-page-parse-on-need.md b/.changeset/cli-search-page-parse-on-need.md
new file mode 100644
index 0000000000..476bd3391c
--- /dev/null
+++ b/.changeset/cli-search-page-parse-on-need.md
@@ -0,0 +1,7 @@
+---
+'@astryxdesign/cli': patch
+---
+
+[perf] `astryx search` and `astryx build` parse a page template's source for the components it renders only when the query could match one of them. A query no page renders parses only the few pages whose source holds an escape, and a first search for it uses about 20% less CPU. A query whose words name components on most pages still parses most of them and can take up to about 4% more. Results are unchanged.
+
+@vjeux
diff --git a/packages/cli/api/search/_candidate-score.mjs b/packages/cli/api/search/_candidate-score.mjs
index 0ce426ee0f..1297820143 100644
--- a/packages/cli/api/search/_candidate-score.mjs
+++ b/packages/cli/api/search/_candidate-score.mjs
@@ -94,6 +94,19 @@ function headOfLetters(term, name) {
/** How many of a family's members a query naming the family lists. */
const FAMILY_MEMBERS = 3;
+/**
+ * A derived name's score for a term: 60 for the same word, 40 for a word the
+ * term starts, else 0.
+ * @param {string} term - Lowercased search term.
+ * @param {string} kw - The derived name as authored.
+ * @returns {number}
+ */
+function weakKeywordScore(term, kw) {
+ const kwLower = String(kw).toLowerCase();
+ if (kwLower === term || sameWord(term, kwLower)) return 60;
+ return startsAWordOf(term, kw) ? 40 : 0;
+}
+
/**
* Score a single candidate against the search term across name, keywords,
* and prose signals. Returns the best (highest) score plus a human reason,
@@ -107,6 +120,8 @@ const FAMILY_MEMBERS = 3;
* also matches typed as words: `command palette` is CommandPalette.
* @param {string[]} [candidate.keywords] - Authored intent (componentsUsed, category words).
* @param {string[]} [candidate.weakKeywords] - Derived signal (components a page renders).
+ * @param {string[]} [candidate.weakKeywordBound] - Every name `weakKeywords`
+ * can hold, when reading them is costly: they are read only if one matches.
* @param {string} [candidate.description]
* @param {string[]} [candidate.prose] - Extra free-text blobs (doc section text, best practices).
* @param {string[]} [candidate.guidance] - Usage guidance (features, best practices) — scored a tier below description.
@@ -118,20 +133,20 @@ const FAMILY_MEMBERS = 3;
*/
export function scoreCandidate(
term,
- {
+ candidate,
+ {fuzzy = true, joins = true, part = false, synonym = false} = {},
+) {
+ const {
name,
aliases = [],
domain,
keywords = [],
- weakKeywords = [],
description = '',
prose = [],
guidance = [],
_family,
_familyIndex,
- },
- {fuzzy = true, joins = true, part = false, synonym = false} = {},
-) {
+ } = candidate;
let best = 0;
let reason = '';
/** @type {'partial' | 'keyword' | false} */
@@ -277,13 +292,14 @@ export function scoreCandidate(
// confident-match gate on its own; it still counts toward multi-term
// coverage, which is where "this page renders that" is real evidence.
// No Levenshtein tier — fuzzy matching a derived signal is pure noise.
- for (const kw of weakKeywords) {
- const kwLower = String(kw).toLowerCase();
- if (kwLower === term || sameWord(term, kwLower)) {
- consider(60, `renders ${kw}`);
- continue;
+ // A bound skips the read when no name the page can yield matches the term,
+ // and then no weak keyword can match it either.
+ const bound = candidate.weakKeywordBound;
+ if (!bound || bound.some(kw => weakKeywordScore(term, kw) > 0)) {
+ for (const kw of candidate.weakKeywords ?? []) {
+ const score = weakKeywordScore(term, kw);
+ if (score > 0) consider(score, `renders ${kw}`);
}
- if (startsAWordOf(term, kw)) consider(40, `renders ${kw}`);
}
// ── Prose / description / guidance signals (stem-tolerant whole word) ──
diff --git a/packages/cli/api/search/_gather-templates.mjs b/packages/cli/api/search/_gather-templates.mjs
index 863e896124..275db197b9 100644
--- a/packages/cli/api/search/_gather-templates.mjs
+++ b/packages/cli/api/search/_gather-templates.mjs
@@ -4,10 +4,15 @@
* @file Search's template and theme candidates. Imported only by the adapter.
*/
+import fs from 'node:fs';
import {CORE_PACKAGE} from '../../foundation/discovery/component-discovery.mjs';
-import {discoverTemplates, extractComponents} from '../template/template.mjs';
+import {discoverTemplates} from '../template/template.mjs';
import {listAvailableThemes} from '../theme/_adapter.mjs';
-import {templateLookupIds} from '../../foundation/discovery/template-adapter.mjs';
+import {
+ componentNameBound,
+ componentsInSource,
+ templateLookupIds,
+} from '../../foundation/discovery/template-adapter.mjs';
import {sameWord, wordsOf} from './_words.mjs';
/** @typedef {import('./_lexical.mjs').Candidate} Candidate */
@@ -66,12 +71,29 @@ export async function gatherTemplates(cwd) {
? [...t.componentsUsed]
: [];
keywords.push(...templateLookupIds(t).filter(id => id !== commandName));
- /** @type {string[]} */
- let weakKeywords = [];
+ /** @type {() => string[]} */
+ let derive = () => [];
+ /** @type {string[] | null} */
+ let weakKeywordBound = null;
if (t.type === 'page') {
if (t.filePath) {
try {
- weakKeywords = extractComponents(t.filePath);
+ const source = fs.readFileSync(t.filePath, 'utf-8');
+ // Parsing the source costs more than the rest of the page's gather.
+ // It waits until a term could match a name the source can yield.
+ /** @type {string[] | undefined} */
+ let derived;
+ derive = () => {
+ if (derived === undefined) {
+ try {
+ derived = componentsInSource(source);
+ } catch {
+ derived = [];
+ }
+ }
+ return derived;
+ };
+ weakKeywordBound = componentNameBound(source);
} catch {
// Best-effort: skip keyword enrichment if the source can't be read.
}
@@ -83,7 +105,10 @@ export async function gatherTemplates(cwd) {
domain: 'template',
name: commandName,
keywords,
- weakKeywords,
+ get weakKeywords() {
+ return derive();
+ },
+ ...(weakKeywordBound ? {weakKeywordBound} : {}),
description: t.description || '',
// A template's own keywords name the many ideas it serves, so they score
// as its prose does, not as a name: one of them matching one word of a
diff --git a/packages/cli/api/search/_gather-templates.test.mjs b/packages/cli/api/search/_gather-templates.test.mjs
new file mode 100644
index 0000000000..b437980aa5
--- /dev/null
+++ b/packages/cli/api/search/_gather-templates.test.mjs
@@ -0,0 +1,259 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file A page template's weak keywords are read from its syntax only when a
+ * term could match one of them. The bound that decides it never changes a
+ * score or a ranking.
+ *
+ * @input the repository's page templates, and small sources for the edges
+ * @output equal scores and rankings with and without the bound; parse counts
+ * @position packages/cli/api/search — the template gatherer
+ */
+
+import {afterEach, beforeAll, describe, expect, it, vi} from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import {createRequire} from 'node:module';
+import {fileURLToPath} from 'node:url';
+import {
+ componentNameBound,
+ componentsInSource,
+} from '../../foundation/discovery/template-adapter.mjs';
+import {gatherTemplates} from './_gather-templates.mjs';
+import {scoreCandidate} from './_candidate-score.mjs';
+import {rankLexical} from './_lexical.mjs';
+import {wordsOf} from './_words.mjs';
+
+const REPO = fs.realpathSync(
+ path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..'),
+);
+const babel = createRequire(import.meta.url)('@babel/parser');
+
+/** @type {any[]} */
+let templates;
+/** @type {any[]} */
+let pages;
+
+beforeAll(async () => {
+ templates = await gatherTemplates(REPO);
+ pages = templates.filter(t => t._kind === 'page');
+});
+
+afterEach(() => vi.restoreAllMocks());
+
+/**
+ * The same page with its weak keywords read up front and no bound.
+ * @param {any} page
+ */
+function unbounded(page) {
+ const {weakKeywordBound: _bound, ...rest} = page;
+ return {...rest, weakKeywords: [...page.weakKeywords]};
+}
+
+/** Terms that match the pages' derived names, nearly match them, or miss. */
+function termsFor() {
+ const words = new Set(['zzqqxx', 'quokka', 'button', 'buttons', 'table']);
+ for (const word of ['chart', 'dashboard', 'settings', 'list', 'card']) {
+ words.add(word);
+ }
+ for (const page of pages.slice(0, 12)) {
+ for (const name of page.weakKeywords) {
+ for (const word of wordsOf(name)) {
+ words.add(word);
+ words.add(`${word}s`);
+ if (word.length > 5) words.add(word.slice(0, 5));
+ }
+ }
+ }
+ return [...words];
+}
+
+describe('the weak-keyword bound of a page template', () => {
+ it('lets the gather and a term no page renders skip the parse', async () => {
+ const parse = vi.spyOn(babel, 'parse');
+ const gathered = await gatherTemplates(REPO);
+ const gatheredPages = gathered.filter(t => t._kind === 'page');
+ expect(gatheredPages.length).toBeGreaterThan(20);
+ expect(parse).not.toHaveBeenCalled();
+ for (const page of gatheredPages) scoreCandidate('zzqqxx', page);
+ // Only a page whose source defeats the bound (an escape) is parsed.
+ const unboundedPages = gatheredPages.filter(p => !p.weakKeywordBound);
+ expect(parse).toHaveBeenCalledTimes(unboundedPages.length);
+ expect(unboundedPages.length).toBeLessThan(gatheredPages.length / 4);
+ });
+
+ it('holds every derived name of every page', () => {
+ for (const page of pages) {
+ if (!page.weakKeywordBound) continue;
+ for (const name of page.weakKeywords) {
+ expect(page.weakKeywordBound, page.name).toContain(name);
+ }
+ }
+ });
+
+ it('gives every term the same score as reading the names up front', () => {
+ const terms = termsFor();
+ expect(terms.length).toBeGreaterThan(40);
+ for (const page of pages) {
+ const plain = unbounded(page);
+ for (const term of terms) {
+ expect(scoreCandidate(term, page), `${page.name}: ${term}`).toEqual(
+ scoreCandidate(term, plain),
+ );
+ }
+ }
+ });
+
+ it('ranks templates the same as reading the names up front', () => {
+ const plain = templates.map(t => (t._kind === 'page' ? unbounded(t) : t));
+ const catalog = /** @param {any[]} all */ all => ({
+ all,
+ components: [],
+ searchedComponents: false,
+ outside: false,
+ coreSkipped: null,
+ });
+ for (const query of [
+ 'settings page with tabs',
+ 'data table with filters',
+ 'kanban board',
+ 'buttons',
+ 'zzqqxx',
+ 'chat',
+ ]) {
+ const options = {cwd: REPO, type: 'template', limit: 50};
+ expect(rankLexical(query, options, catalog(templates)), query).toEqual(
+ rankLexical(query, options, catalog(plain)),
+ );
+ }
+ });
+});
+
+describe('componentNameBound', () => {
+ /** @param {string} src */
+ const expectBound = src => {
+ const bound = componentNameBound(src);
+ expect(bound).not.toBeNull();
+ for (const name of componentsInSource(src)) expect(bound).toContain(name);
+ return bound;
+ };
+
+ it('holds the names of a page with no Astryx import', () => {
+ expectBound(
+ "import {Foo as Bar} from 'other';\nexport default () => <Bar><Baz /></Bar>;",
+ );
+ });
+
+ it('holds the imported name a renamed import gives a tag', () => {
+ const bound = expectBound(
+ "import {Avatar as Av} from '@astryxdesign/core';\nexport default () => <Av />;",
+ );
+ expect(bound).toContain('Avatar');
+ });
+
+ it('reads a name imported as a string, and comments between the words', () => {
+ expect(
+ expectBound(
+ "import {'Avatar' as Av} from '@astryxdesign/core';\nexport default () => <Av />;",
+ ),
+ ).toContain('Avatar');
+ expect(
+ expectBound(
+ "import {Tokenizer /* a */ as // b\n Tok} from '@astryxdesign/core';\nexport default () => <Tok />;",
+ ),
+ ).toContain('Tokenizer');
+ });
+
+ it('reads a renamed import after a comment in an earlier import', () => {
+ expect(
+ expectBound(
+ [
+ "import {Card /* card */} from '@astryxdesign/core';",
+ "import {Avatar /* user */ as Av} from '@astryxdesign/core';",
+ 'export default () => (',
+ ' <Card>',
+ ' <Av />',
+ ' </Card>',
+ ');',
+ ].join('\n'),
+ ),
+ ).toContain('Avatar');
+ });
+
+ it('steps past a pair that starts with an astral letter', () => {
+ expect(
+ expectBound(
+ [
+ "import {Beta} from '@astryxdesign/core';",
+ '// 𝐀 as Beta',
+ 'export default () => <Beta />;',
+ ].join('\n'),
+ ),
+ ).toContain('Beta');
+ });
+
+ it('reads a run of line comments that hold links in one pass', () => {
+ const links = Array.from(
+ {length: 40},
+ (_, i) => `// See https://example.com/notes/${i} for more.`,
+ );
+ expect(
+ expectBound(
+ [
+ "import {Card} from '@astryxdesign/core';",
+ 'const label = title',
+ ...links,
+ 'export default () => <Card />;',
+ ].join('\n'),
+ ),
+ ).toContain('Card');
+ });
+
+ it('leaves a source with a very long run of comments to the parse', () => {
+ const notes = Array.from(
+ {length: 1000},
+ (_, i) => `// note ${i} about thing`,
+ );
+ const src = [
+ "import {Card} from '@astryxdesign/core';",
+ 'const label = title',
+ ...notes,
+ 'export default () => <Card />;',
+ ].join('\n');
+ expect(componentNameBound(src)).toBeNull();
+ });
+
+ it('reads a renamed import after an "as /*" in a comment', () => {
+ expect(
+ expectBound(
+ [
+ '// Shown as /* in the docs.',
+ "import {Avatar as Av} from '@astryxdesign/core';",
+ '/** A later doc comment. */',
+ 'export default () => <Av />;',
+ ].join('\n'),
+ ),
+ ).toContain('Avatar');
+ });
+
+ it('keeps a name the parse drops as page-local', () => {
+ expect(
+ expectBound(
+ 'function TimelineSection() {\n return null;\n}\nexport default () => <TimelineSection />;',
+ ),
+ ).toContain('Timeline');
+ });
+
+ it('is null for a source with an escape it cannot read', () => {
+ expect(
+ componentNameBound(
+ "import {\\u0041vatar as Av} from '@astryxdesign/core';\nexport default () => <Av />;",
+ ),
+ ).toBeNull();
+ expect(
+ componentNameBound(
+ "import {'Av\\x61tar' as Av} from '@astryxdesign/core';\nexport default () => <Av />;",
+ ),
+ ).toBeNull();
+ });
+});
diff --git a/packages/cli/api/search/_lexical.mjs b/packages/cli/api/search/_lexical.mjs
index b60bc04314..148e6f6c93 100644
--- a/packages/cli/api/search/_lexical.mjs
+++ b/packages/cli/api/search/_lexical.mjs
@@ -73,6 +73,8 @@ import {toResult, searchedComponentsOf} from './_result.mjs';
* author lists in `keywords`, without the code terms and table cells its
* other keywords add.
* @property {string[]} [weakKeywords]
+ * @property {string[]} [weakKeywordBound] - Every name `weakKeywords` can
+ * hold, for a page whose weak keywords cost a parse to read.
* @property {string} [description]
* @property {string[]} [prose]
* @property {string[]} [guidance]
diff --git a/packages/cli/foundation/discovery/template-adapter.mjs b/packages/cli/foundation/discovery/template-adapter.mjs
index af4b22975b..05a55dd691 100644
--- a/packages/cli/foundation/discovery/template-adapter.mjs
+++ b/packages/cli/foundation/discovery/template-adapter.mjs
@@ -1976,6 +1976,41 @@ export async function findShowcase(componentName, cwd, options) {
/** The npm scope of every Astryx package. */
const ASTRYX_PACKAGE_SCOPE = '@astryxdesign/';
+/** A JSX opening tag; componentsInSource says why the pattern looks so. */
+const JSX_TAG = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;
+
+/** Comments and line breaks that may sit between two words of an import. */
+// A block comment ends at its first `*/`, and a line comment at its line's
+// end. So each comment has one reading, and a failed match cannot backtrack
+// through every split of a comment run.
+const SPACE_OR_COMMENT = String.raw`\s|\/\*(?:[^*]|\*(?!\/))*\*\/|\/\/[^\n\r\u2028\u2029]*(?![^\n\r\u2028\u2029])`;
+const BETWEEN = `(?:${SPACE_OR_COMMENT})*`;
+const GAP = new RegExp(`(?:${SPACE_OR_COMMENT})+`, 'g');
+// The pair scan can start again at each word of a run of spaces and
+// comments, so a run costs up to its length squared. The longest run in the
+// bundled templates is about 10 KB.
+const MAX_GAP = 16 * 1024;
+
+/**
+ * One `imported as local` pair. The imported name is an identifier (group 1)
+ * or a string (group 3); the local name is group 4.
+ */
+const ALIAS = new RegExp(
+ String.raw`(?:(?<![\p{ID_Continue}$])([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)|(['"])((?:\\[\s\S]|(?!\2)[^\\\n])*)\2)` +
+ BETWEEN +
+ String.raw`\bas\b` +
+ BETWEEN +
+ String.raw`([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)`,
+ 'gu',
+);
+// What follows each `as`: the pair scan reads the same text after its `as`.
+const AS_LOCAL = new RegExp(
+ String.raw`\bas\b` +
+ BETWEEN +
+ String.raw`([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)`,
+ 'gu',
+);
+
const UBIQUITOUS = new Set([
'Text',
'Heading',
@@ -2079,7 +2114,7 @@ export function componentsInSource(src) {
// and `Record<string, Phase>` no longer do. Those were being indexed as
// rendered components, putting type names like `Record`, `SVGProps` and
// `ReadonlySet` into template keyword and "components used" output.
- const tagRegex = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;
+ const tagRegex = new RegExp(JSX_TAG);
/** @type {string[]} */
const matches = [];
let m;
@@ -2093,11 +2128,23 @@ export function componentsInSource(src) {
// strings and type annotations cannot add or hide a name.
const {declared: localDeclNames, astryxImports} = topLevelBindings(src);
+ return catalogNames(
+ matches
+ .filter(n => !localDeclNames.has(n))
+ .map(n => astryxImports.get(n) ?? n),
+ );
+}
+
+/**
+ * Rendered component names as the catalog lists them: without theme wrappers
+ * and ubiquitous parts, each part suffix stripped, each name once, sorted.
+ * @param {string[]} names
+ * @returns {string[]}
+ */
+function catalogNames(names) {
return [
...new Set(
- matches
- .filter(n => !localDeclNames.has(n))
- .map(n => astryxImports.get(n) ?? n)
+ names
.filter(n => !['Theme', 'ThemeProvider'].includes(n))
.filter(n => !UBIQUITOUS.has(n))
.map(n =>
@@ -2112,3 +2159,46 @@ export function componentsInSource(src) {
),
].sort();
}
+
+/**
+ * Every name componentsInSource can return for `src`, read without parsing
+ * it, or null when it cannot be bounded so. It keeps the tag names a parse
+ * would drop as page-local. It adds each name an `imported as local` pair
+ * makes a tag stand for, wherever that text appears. An escaped name in an
+ * identifier or a string defeats the reading, so then it is null.
+ * @param {string} src
+ * @returns {string[] | null}
+ */
+export function componentNameBound(src) {
+ if (src.includes('\\u')) return null;
+ for (const gap of src.matchAll(GAP)) {
+ if (gap[0].length > MAX_GAP) return null;
+ }
+ const tags = Array.from(src.matchAll(JSX_TAG), m => m[2]);
+ const local = new Set(tags);
+ // A pair adds a name only when its local name is a tag. Most sources have
+ // no `as` before a tag's name, so they skip the slower pair scan. Each
+ // match restarts one character on: a `/*` in a comment can hide a pair.
+ const asLocal = new RegExp(AS_LOCAL);
+ let pairs = false;
+ for (let m = asLocal.exec(src); m && !pairs; m = asLocal.exec(src)) {
+ asLocal.lastIndex = m.index + 1;
+ pairs = local.has(m[1]);
+ }
+ if (!pairs) return catalogNames(tags);
+ /** @type {string[]} */
+ const imported = [];
+ // Each match restarts one character on, so no pair hides inside another.
+ const alias = new RegExp(ALIAS);
+ for (let m = alias.exec(src); m; m = alias.exec(src)) {
+ // Step one code point: a `u` regex moves an index inside a surrogate
+ // pair back to the pair's start.
+ alias.lastIndex =
+ m.index + ((src.codePointAt(m.index) ?? 0) > 0xffff ? 2 : 1);
+ if (!local.has(m[4])) continue;
+ const name = m[1] ?? m[3];
+ if (name.includes('\\')) return null;
+ imported.push(name);
+ }
+ return catalogNames([...tags, ...imported]);
+}