An open source design system that's fully customizable and agent ready
TypeScript
perf(cli): search parses a page template only when a term can match it (#9871)
Christopher Chedeau442f2ddd4f05
diff --git a/.changeset/cli-search-page-parse-on-need.md b/.changeset/cli-search-page-parse-on-need.mdnew file mode 100644index 0000000000..476bd3391c--- /dev/null+++ b/.changeset/cli-search-page-parse-on-need.md@@ -0,0 +1,7 @@+---+'@astryxdesign/cli': patch+---++[perf] `astryx search` and `astryx build` parse a page template's source for the components it renders only when the query could match one of them. A query no page renders parses only the few pages whose source holds an escape, and a first search for it uses about 20% less CPU. A query whose words name components on most pages still parses most of them and can take up to about 4% more. Results are unchanged.++@vjeuxdiff --git a/packages/cli/api/search/_candidate-score.mjs b/packages/cli/api/search/_candidate-score.mjsindex 0ce426ee0f..1297820143 100644--- a/packages/cli/api/search/_candidate-score.mjs+++ b/packages/cli/api/search/_candidate-score.mjs@@ -94,6 +94,19 @@ function headOfLetters(term, name) {/** How many of a family's members a query naming the family lists. */const FAMILY_MEMBERS = 3;+/**+ * A derived name's score for a term: 60 for the same word, 40 for a word the+ * term starts, else 0.+ * @param {string} term - Lowercased search term.+ * @param {string} kw - The derived name as authored.+ * @returns {number}+ */+function weakKeywordScore(term, kw) {+ const kwLower = String(kw).toLowerCase();+ if (kwLower === term || sameWord(term, kwLower)) return 60;+ return startsAWordOf(term, kw) ? 40 : 0;+}+/*** Score a single candidate against the search term across name, keywords,* and prose signals. Returns the best (highest) score plus a human reason,@@ -107,6 +120,8 @@ const FAMILY_MEMBERS = 3;* also matches typed as words: `command palette` is CommandPalette.* @param {string[]} [candidate.keywords] - Authored intent (componentsUsed, category words).* @param {string[]} [candidate.weakKeywords] - Derived signal (components a page renders).+ * @param {string[]} [candidate.weakKeywordBound] - Every name `weakKeywords`+ * can hold, when reading them is costly: they are read only if one matches.* @param {string} [candidate.description]* @param {string[]} [candidate.prose] - Extra free-text blobs (doc section text, best practices).* @param {string[]} [candidate.guidance] - Usage guidance (features, best practices) — scored a tier below description.@@ -118,20 +133,20 @@ const FAMILY_MEMBERS = 3;*/export function scoreCandidate(term,- {+ candidate,+ {fuzzy = true, joins = true, part = false, synonym = false} = {},+) {+ const {name,aliases = [],domain,keywords = [],- weakKeywords = [],description = '',prose = [],guidance = [],_family,_familyIndex,- },- {fuzzy = true, joins = true, part = false, synonym = false} = {},-) {+ } = candidate;let best = 0;let reason = '';/** @type {'partial' | 'keyword' | false} */@@ -277,13 +292,14 @@ export function scoreCandidate(// confident-match gate on its own; it still counts toward multi-term// coverage, which is where "this page renders that" is real evidence.// No Levenshtein tier — fuzzy matching a derived signal is pure noise.- for (const kw of weakKeywords) {- const kwLower = String(kw).toLowerCase();- if (kwLower === term || sameWord(term, kwLower)) {- consider(60, `renders ${kw}`);- continue;+ // A bound skips the read when no name the page can yield matches the term,+ // and then no weak keyword can match it either.+ const bound = candidate.weakKeywordBound;+ if (!bound || bound.some(kw => weakKeywordScore(term, kw) > 0)) {+ for (const kw of candidate.weakKeywords ?? []) {+ const score = weakKeywordScore(term, kw);+ if (score > 0) consider(score, `renders ${kw}`);}- if (startsAWordOf(term, kw)) consider(40, `renders ${kw}`);}// ── Prose / description / guidance signals (stem-tolerant whole word) ──diff --git a/packages/cli/api/search/_gather-templates.mjs b/packages/cli/api/search/_gather-templates.mjsindex 863e896124..275db197b9 100644--- a/packages/cli/api/search/_gather-templates.mjs+++ b/packages/cli/api/search/_gather-templates.mjs@@ -4,10 +4,15 @@* @file Search's template and theme candidates. Imported only by the adapter.*/+import fs from 'node:fs';import {CORE_PACKAGE} from '../../foundation/discovery/component-discovery.mjs';-import {discoverTemplates, extractComponents} from '../template/template.mjs';+import {discoverTemplates} from '../template/template.mjs';import {listAvailableThemes} from '../theme/_adapter.mjs';-import {templateLookupIds} from '../../foundation/discovery/template-adapter.mjs';+import {+ componentNameBound,+ componentsInSource,+ templateLookupIds,+} from '../../foundation/discovery/template-adapter.mjs';import {sameWord, wordsOf} from './_words.mjs';/** @typedef {import('./_lexical.mjs').Candidate} Candidate */@@ -66,12 +71,29 @@ export async function gatherTemplates(cwd) {? [...t.componentsUsed]: [];keywords.push(...templateLookupIds(t).filter(id => id !== commandName));- /** @type {string[]} */- let weakKeywords = [];+ /** @type {() => string[]} */+ let derive = () => [];+ /** @type {string[] | null} */+ let weakKeywordBound = null;if (t.type === 'page') {if (t.filePath) {try {- weakKeywords = extractComponents(t.filePath);+ const source = fs.readFileSync(t.filePath, 'utf-8');+ // Parsing the source costs more than the rest of the page's gather.+ // It waits until a term could match a name the source can yield.+ /** @type {string[] | undefined} */+ let derived;+ derive = () => {+ if (derived === undefined) {+ try {+ derived = componentsInSource(source);+ } catch {+ derived = [];+ }+ }+ return derived;+ };+ weakKeywordBound = componentNameBound(source);} catch {// Best-effort: skip keyword enrichment if the source can't be read.}@@ -83,7 +105,10 @@ export async function gatherTemplates(cwd) {domain: 'template',name: commandName,keywords,- weakKeywords,+ get weakKeywords() {+ return derive();+ },+ ...(weakKeywordBound ? {weakKeywordBound} : {}),description: t.description || '',// A template's own keywords name the many ideas it serves, so they score// as its prose does, not as a name: one of them matching one word of adiff --git a/packages/cli/api/search/_gather-templates.test.mjs b/packages/cli/api/search/_gather-templates.test.mjsnew file mode 100644index 0000000000..b437980aa5--- /dev/null+++ b/packages/cli/api/search/_gather-templates.test.mjs@@ -0,0 +1,259 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file A page template's weak keywords are read from its syntax only when a+ * term could match one of them. The bound that decides it never changes a+ * score or a ranking.+ *+ * @input the repository's page templates, and small sources for the edges+ * @output equal scores and rankings with and without the bound; parse counts+ * @position packages/cli/api/search — the template gatherer+ */++import {afterEach, beforeAll, describe, expect, it, vi} from 'vitest';+import * as fs from 'node:fs';+import * as path from 'node:path';+import {createRequire} from 'node:module';+import {fileURLToPath} from 'node:url';+import {+ componentNameBound,+ componentsInSource,+} from '../../foundation/discovery/template-adapter.mjs';+import {gatherTemplates} from './_gather-templates.mjs';+import {scoreCandidate} from './_candidate-score.mjs';+import {rankLexical} from './_lexical.mjs';+import {wordsOf} from './_words.mjs';++const REPO = fs.realpathSync(+ path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..'),+);+const babel = createRequire(import.meta.url)('@babel/parser');++/** @type {any[]} */+let templates;+/** @type {any[]} */+let pages;++beforeAll(async () => {+ templates = await gatherTemplates(REPO);+ pages = templates.filter(t => t._kind === 'page');+});++afterEach(() => vi.restoreAllMocks());++/**+ * The same page with its weak keywords read up front and no bound.+ * @param {any} page+ */+function unbounded(page) {+ const {weakKeywordBound: _bound, ...rest} = page;+ return {...rest, weakKeywords: [...page.weakKeywords]};+}++/** Terms that match the pages' derived names, nearly match them, or miss. */+function termsFor() {+ const words = new Set(['zzqqxx', 'quokka', 'button', 'buttons', 'table']);+ for (const word of ['chart', 'dashboard', 'settings', 'list', 'card']) {+ words.add(word);+ }+ for (const page of pages.slice(0, 12)) {+ for (const name of page.weakKeywords) {+ for (const word of wordsOf(name)) {+ words.add(word);+ words.add(`${word}s`);+ if (word.length > 5) words.add(word.slice(0, 5));+ }+ }+ }+ return [...words];+}++describe('the weak-keyword bound of a page template', () => {+ it('lets the gather and a term no page renders skip the parse', async () => {+ const parse = vi.spyOn(babel, 'parse');+ const gathered = await gatherTemplates(REPO);+ const gatheredPages = gathered.filter(t => t._kind === 'page');+ expect(gatheredPages.length).toBeGreaterThan(20);+ expect(parse).not.toHaveBeenCalled();+ for (const page of gatheredPages) scoreCandidate('zzqqxx', page);+ // Only a page whose source defeats the bound (an escape) is parsed.+ const unboundedPages = gatheredPages.filter(p => !p.weakKeywordBound);+ expect(parse).toHaveBeenCalledTimes(unboundedPages.length);+ expect(unboundedPages.length).toBeLessThan(gatheredPages.length / 4);+ });++ it('holds every derived name of every page', () => {+ for (const page of pages) {+ if (!page.weakKeywordBound) continue;+ for (const name of page.weakKeywords) {+ expect(page.weakKeywordBound, page.name).toContain(name);+ }+ }+ });++ it('gives every term the same score as reading the names up front', () => {+ const terms = termsFor();+ expect(terms.length).toBeGreaterThan(40);+ for (const page of pages) {+ const plain = unbounded(page);+ for (const term of terms) {+ expect(scoreCandidate(term, page), `${page.name}: ${term}`).toEqual(+ scoreCandidate(term, plain),+ );+ }+ }+ });++ it('ranks templates the same as reading the names up front', () => {+ const plain = templates.map(t => (t._kind === 'page' ? unbounded(t) : t));+ const catalog = /** @param {any[]} all */ all => ({+ all,+ components: [],+ searchedComponents: false,+ outside: false,+ coreSkipped: null,+ });+ for (const query of [+ 'settings page with tabs',+ 'data table with filters',+ 'kanban board',+ 'buttons',+ 'zzqqxx',+ 'chat',+ ]) {+ const options = {cwd: REPO, type: 'template', limit: 50};+ expect(rankLexical(query, options, catalog(templates)), query).toEqual(+ rankLexical(query, options, catalog(plain)),+ );+ }+ });+});++describe('componentNameBound', () => {+ /** @param {string} src */+ const expectBound = src => {+ const bound = componentNameBound(src);+ expect(bound).not.toBeNull();+ for (const name of componentsInSource(src)) expect(bound).toContain(name);+ return bound;+ };++ it('holds the names of a page with no Astryx import', () => {+ expectBound(+ "import {Foo as Bar} from 'other';\nexport default () => <Bar><Baz /></Bar>;",+ );+ });++ it('holds the imported name a renamed import gives a tag', () => {+ const bound = expectBound(+ "import {Avatar as Av} from '@astryxdesign/core';\nexport default () => <Av />;",+ );+ expect(bound).toContain('Avatar');+ });++ it('reads a name imported as a string, and comments between the words', () => {+ expect(+ expectBound(+ "import {'Avatar' as Av} from '@astryxdesign/core';\nexport default () => <Av />;",+ ),+ ).toContain('Avatar');+ expect(+ expectBound(+ "import {Tokenizer /* a */ as // b\n Tok} from '@astryxdesign/core';\nexport default () => <Tok />;",+ ),+ ).toContain('Tokenizer');+ });++ it('reads a renamed import after a comment in an earlier import', () => {+ expect(+ expectBound(+ [+ "import {Card /* card */} from '@astryxdesign/core';",+ "import {Avatar /* user */ as Av} from '@astryxdesign/core';",+ 'export default () => (',+ ' <Card>',+ ' <Av />',+ ' </Card>',+ ');',+ ].join('\n'),+ ),+ ).toContain('Avatar');+ });++ it('steps past a pair that starts with an astral letter', () => {+ expect(+ expectBound(+ [+ "import {Beta} from '@astryxdesign/core';",+ '// 𝐀 as Beta',+ 'export default () => <Beta />;',+ ].join('\n'),+ ),+ ).toContain('Beta');+ });++ it('reads a run of line comments that hold links in one pass', () => {+ const links = Array.from(+ {length: 40},+ (_, i) => `// See https://example.com/notes/${i} for more.`,+ );+ expect(+ expectBound(+ [+ "import {Card} from '@astryxdesign/core';",+ 'const label = title',+ ...links,+ 'export default () => <Card />;',+ ].join('\n'),+ ),+ ).toContain('Card');+ });++ it('leaves a source with a very long run of comments to the parse', () => {+ const notes = Array.from(+ {length: 1000},+ (_, i) => `// note ${i} about thing`,+ );+ const src = [+ "import {Card} from '@astryxdesign/core';",+ 'const label = title',+ ...notes,+ 'export default () => <Card />;',+ ].join('\n');+ expect(componentNameBound(src)).toBeNull();+ });++ it('reads a renamed import after an "as /*" in a comment', () => {+ expect(+ expectBound(+ [+ '// Shown as /* in the docs.',+ "import {Avatar as Av} from '@astryxdesign/core';",+ '/** A later doc comment. */',+ 'export default () => <Av />;',+ ].join('\n'),+ ),+ ).toContain('Avatar');+ });++ it('keeps a name the parse drops as page-local', () => {+ expect(+ expectBound(+ 'function TimelineSection() {\n return null;\n}\nexport default () => <TimelineSection />;',+ ),+ ).toContain('Timeline');+ });++ it('is null for a source with an escape it cannot read', () => {+ expect(+ componentNameBound(+ "import {\\u0041vatar as Av} from '@astryxdesign/core';\nexport default () => <Av />;",+ ),+ ).toBeNull();+ expect(+ componentNameBound(+ "import {'Av\\x61tar' as Av} from '@astryxdesign/core';\nexport default () => <Av />;",+ ),+ ).toBeNull();+ });+});diff --git a/packages/cli/api/search/_lexical.mjs b/packages/cli/api/search/_lexical.mjsindex b60bc04314..148e6f6c93 100644--- a/packages/cli/api/search/_lexical.mjs+++ b/packages/cli/api/search/_lexical.mjs@@ -73,6 +73,8 @@ import {toResult, searchedComponentsOf} from './_result.mjs';* author lists in `keywords`, without the code terms and table cells its* other keywords add.* @property {string[]} [weakKeywords]+ * @property {string[]} [weakKeywordBound] - Every name `weakKeywords` can+ * hold, for a page whose weak keywords cost a parse to read.* @property {string} [description]* @property {string[]} [prose]* @property {string[]} [guidance]diff --git a/packages/cli/foundation/discovery/template-adapter.mjs b/packages/cli/foundation/discovery/template-adapter.mjsindex af4b22975b..05a55dd691 100644--- a/packages/cli/foundation/discovery/template-adapter.mjs+++ b/packages/cli/foundation/discovery/template-adapter.mjs@@ -1976,6 +1976,41 @@ export async function findShowcase(componentName, cwd, options) {/** The npm scope of every Astryx package. */const ASTRYX_PACKAGE_SCOPE = '@astryxdesign/';+/** A JSX opening tag; componentsInSource says why the pattern looks so. */+const JSX_TAG = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;++/** Comments and line breaks that may sit between two words of an import. */+// A block comment ends at its first `*/`, and a line comment at its line's+// end. So each comment has one reading, and a failed match cannot backtrack+// through every split of a comment run.+const SPACE_OR_COMMENT = String.raw`\s|\/\*(?:[^*]|\*(?!\/))*\*\/|\/\/[^\n\r\u2028\u2029]*(?![^\n\r\u2028\u2029])`;+const BETWEEN = `(?:${SPACE_OR_COMMENT})*`;+const GAP = new RegExp(`(?:${SPACE_OR_COMMENT})+`, 'g');+// The pair scan can start again at each word of a run of spaces and+// comments, so a run costs up to its length squared. The longest run in the+// bundled templates is about 10 KB.+const MAX_GAP = 16 * 1024;++/**+ * One `imported as local` pair. The imported name is an identifier (group 1)+ * or a string (group 3); the local name is group 4.+ */+const ALIAS = new RegExp(+ String.raw`(?:(?<![\p{ID_Continue}$])([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)|(['"])((?:\\[\s\S]|(?!\2)[^\\\n])*)\2)` ++ BETWEEN ++ String.raw`\bas\b` ++ BETWEEN ++ String.raw`([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)`,+ 'gu',+);+// What follows each `as`: the pair scan reads the same text after its `as`.+const AS_LOCAL = new RegExp(+ String.raw`\bas\b` ++ BETWEEN ++ String.raw`([\p{ID_Start}$_][\p{ID_Continue}$\u200C\u200D]*)`,+ 'gu',+);+const UBIQUITOUS = new Set(['Text','Heading',@@ -2079,7 +2114,7 @@ export function componentsInSource(src) {// and `Record<string, Phase>` no longer do. Those were being indexed as// rendered components, putting type names like `Record`, `SVGProps` and// `ReadonlySet` into template keyword and "components used" output.- const tagRegex = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;+ const tagRegex = new RegExp(JSX_TAG);/** @type {string[]} */const matches = [];let m;@@ -2093,11 +2128,23 @@ export function componentsInSource(src) {// strings and type annotations cannot add or hide a name.const {declared: localDeclNames, astryxImports} = topLevelBindings(src);+ return catalogNames(+ matches+ .filter(n => !localDeclNames.has(n))+ .map(n => astryxImports.get(n) ?? n),+ );+}++/**+ * Rendered component names as the catalog lists them: without theme wrappers+ * and ubiquitous parts, each part suffix stripped, each name once, sorted.+ * @param {string[]} names+ * @returns {string[]}+ */+function catalogNames(names) {return [...new Set(- matches- .filter(n => !localDeclNames.has(n))- .map(n => astryxImports.get(n) ?? n)+ names.filter(n => !['Theme', 'ThemeProvider'].includes(n)).filter(n => !UBIQUITOUS.has(n)).map(n =>@@ -2112,3 +2159,46 @@ export function componentsInSource(src) {),].sort();}++/**+ * Every name componentsInSource can return for `src`, read without parsing+ * it, or null when it cannot be bounded so. It keeps the tag names a parse+ * would drop as page-local. It adds each name an `imported as local` pair+ * makes a tag stand for, wherever that text appears. An escaped name in an+ * identifier or a string defeats the reading, so then it is null.+ * @param {string} src+ * @returns {string[] | null}+ */+export function componentNameBound(src) {+ if (src.includes('\\u')) return null;+ for (const gap of src.matchAll(GAP)) {+ if (gap[0].length > MAX_GAP) return null;+ }+ const tags = Array.from(src.matchAll(JSX_TAG), m => m[2]);+ const local = new Set(tags);+ // A pair adds a name only when its local name is a tag. Most sources have+ // no `as` before a tag's name, so they skip the slower pair scan. Each+ // match restarts one character on: a `/*` in a comment can hide a pair.+ const asLocal = new RegExp(AS_LOCAL);+ let pairs = false;+ for (let m = asLocal.exec(src); m && !pairs; m = asLocal.exec(src)) {+ asLocal.lastIndex = m.index + 1;+ pairs = local.has(m[1]);+ }+ if (!pairs) return catalogNames(tags);+ /** @type {string[]} */+ const imported = [];+ // Each match restarts one character on, so no pair hides inside another.+ const alias = new RegExp(ALIAS);+ for (let m = alias.exec(src); m; m = alias.exec(src)) {+ // Step one code point: a `u` regex moves an index inside a surrogate+ // pair back to the pair's start.+ alias.lastIndex =+ m.index + ((src.codePointAt(m.index) ?? 0) > 0xffff ? 2 : 1);+ if (!local.has(m[4])) continue;+ const name = m[1] ?? m[3];+ if (name.includes('\\')) return null;+ imported.push(name);+ }+ return catalogNames([...tags, ...imported]);+}