Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(cli): stop build ranking partial query matches above full ones (#5614)

Ernestc6041cfd58ea
diff --git a/.changeset/build-retrieval-scoring.md b/.changeset/build-retrieval-scoring.md
new file mode 100644
index 0000000000..2921c8ec32
--- /dev/null
+++ b/.changeset/build-retrieval-scoring.md
@@ -0,0 +1,7 @@
+---
+'@astryxdesign/cli': patch
+---
+
+[fix] `build`/`search` no longer rank a partially-matched template above one matching every term, no longer index TypeScript generic arguments as rendered components, and no longer treat breadth of rendered components as full-strength relevance.
+
+@ernestt
diff --git a/packages/cli/api/search/search.mjs b/packages/cli/api/search/search.mjs
index 5091f7c350..4660f0ddf6 100644
--- a/packages/cli/api/search/search.mjs
+++ b/packages/cli/api/search/search.mjs
@@ -21,13 +21,24 @@
* 80 name Levenshtein distance 1
* 70 keyword substring / distance 1
* 60 name substring (>=4 chars, >=50% coverage)
+ * 60 exact weak-keyword match
* 50 description / prose mentions the term
* 40 name Levenshtein distance 2
+ * 40 weak-keyword substring
* 30 keyword Levenshtein distance 2
* 20 name Levenshtein distance 3
*
* Name + keyword signals always outweigh description/prose, so an exact match
* sorts above an incidental mention.
+ *
+ * `keywords` carry AUTHORED intent — a block's `componentsUsed`, a page's
+ * `category` words. `weakKeywords` are DERIVED: the components a page template
+ * happens to render, scraped out of its JSX. Derived signal is deliberately
+ * capped below the confident-match gate on its own, because breadth is not
+ * relevance. At full keyword strength every rendered component is an
+ * independent 90-point shot with no penalty for how many a template has, so
+ * the broadest pages (a theme showcase rendering one of everything) win
+ * queries they have nothing to do with.
*/
import {pathToFileURL} from 'node:url';
@@ -53,6 +64,7 @@ import {ERROR_CODES} from '../../foundation/response/error-codes.mjs';
* @property {'component'|'hook'|'doc'|'template'} domain
* @property {string} name
* @property {string[]} [keywords]
+ * @property {string[]} [weakKeywords]
* @property {string} [description]
* @property {string[]} [prose]
* @property {string} [_import]
@@ -222,28 +234,39 @@ export function scoreQuery(term, tokens, candidate) {
// Multi-word natural language: score each content token, counting only
// strong hits, then reward coverage so candidates matching more terms win.
- let sum = 0;
+ let strongest = 0;
let matched = 0;
/** @type {string[]} */
const hitTerms = [];
for (const tok of tokens) {
const h = bestForToken(tok, candidate);
if (h && h.score >= MIN_TOKEN_SCORE) {
- sum += h.score;
+ if (h.score > strongest) strongest = h.score;
matched++;
hitTerms.push(tok);
}
}
if (matched === 0) return full;
- // Reward the AVERAGE strength of the concepts that matched (not divided by
- // total query length — that penalizes verbose / low-fidelity prompts), plus
- // a bonus per additional matched concept and a coverage term. A candidate
- // that matches several of the query's concepts beats one matching a single
- // incidental word.
- const avgMatched = sum / matched;
+ // Base the score on the STRONGEST concept that matched, plus a bonus per
+ // additional matched concept and a coverage term.
+ //
+ // Deliberately not the mean, and deliberately not divided by total query
+ // length. Dividing by total length penalizes verbose / low-fidelity prompts.
+ // Dividing by the number of MATCHED tokens (what this used to do) made the
+ // score non-monotonic: a second, weaker hit could drag the mean down by more
+ // than the coverage bonus added it back, so matching fewer terms well beat
+ // matching more terms partially. Concretely, `build "file browser"` scored
+ // two form wizards matching only "file" at 98, above the actual file browser
+ // matching both terms at 97 — and 98 clears the PAGE_DIRECT gate, so the
+ // wrong template was returned as a confident match.
+ //
+ // Taking the max keeps both properties: a verbose prompt is still scored on
+ // the concepts it did hit, and matching a superset of another candidate's
+ // terms can never score lower, since every term of the expression is
+ // non-decreasing in the set of matched tokens.
const coverage = matched / tokens.length;
- const tokenScore = Math.round(avgMatched + Math.min(matched - 1, 3) * 12 + coverage * 15);
+ const tokenScore = Math.round(strongest + Math.min(matched - 1, 3) * 12 + coverage * 15);
if (full && full.score >= tokenScore) return full;
return {
@@ -260,12 +283,16 @@ export function scoreQuery(term, tokens, candidate) {
* @param {string} term - Lowercased search term.
* @param {object} candidate
* @param {string} candidate.name - Primary identifier (component/hook name, topic, template name).
- * @param {string[]} [candidate.keywords]
+ * @param {string[]} [candidate.keywords] - Authored intent (componentsUsed, category words).
+ * @param {string[]} [candidate.weakKeywords] - Derived signal (components a page renders).
* @param {string} [candidate.description]
* @param {string[]} [candidate.prose] - Extra free-text blobs (doc section text, best practices).
* @returns {{score: number, reason: string} | null}
*/
-export function scoreCandidate(term, {name, keywords = [], description = '', prose = []}) {
+export function scoreCandidate(
+ term,
+ {name, keywords = [], weakKeywords = [], description = '', prose = []},
+) {
let best = 0;
let reason = '';
/**
@@ -314,6 +341,24 @@ export function scoreCandidate(term, {name, keywords = [], description = '', pro
else if (dist === 2) consider(30, `keyword "${kw}" (distance ${dist})`);
}
+ // ── Weak keyword signals (derived, not authored) ─────────────────
+ // Capped at 60 so one incidental component match cannot clear the
+ // confident-match gate on its own; it still counts toward multi-term
+ // coverage, which is where "this page renders that" is real evidence.
+ // No Levenshtein tier — fuzzy matching a derived signal is pure noise.
+ for (const kw of weakKeywords) {
+ const kwLower = String(kw).toLowerCase();
+ if (kwLower === term) {
+ consider(60, `renders ${kw}`);
+ continue;
+ }
+ const s = term.length < kwLower.length ? term : kwLower;
+ const l = term.length < kwLower.length ? kwLower : term;
+ if (s.length >= 4 && l.includes(s) && s.length / l.length >= 0.5) {
+ consider(40, `renders ${kw}`);
+ }
+ }
+
// ── Prose / description signals (stem-tolerant whole word) ──────
// Match the term's stem as a whole word, tolerating plural/gerund suffixes
// so "chart" matches "charts" and "filter" matches "filtering".
@@ -529,24 +574,33 @@ async function gatherTemplates(cwd) {
return [];
}
return templates.map(t => {
- // Blocks ship componentsUsed; page templates don't, so derive them from the
- // source. Category words (e.g. "Dashboard - Analytics") are strong intent
- // signal for pages, which otherwise only index on name + description.
- let keywords = Array.isArray(t.componentsUsed) ? [...t.componentsUsed] : [];
+ // Blocks ship an authored componentsUsed; page templates don't, so derive
+ // them from the source. Category words (e.g. "Dashboard - Analytics") are
+ // strong intent signal for pages, which otherwise only index on name +
+ // description.
+ //
+ // Authored signal and derived signal are kept apart: componentsUsed and
+ // category words are a deliberate statement of what the template is for,
+ // while scraped JSX tags only say what it happens to render. See the
+ // scoring table at the top of this file for why the derived set is capped.
+ const keywords = Array.isArray(t.componentsUsed) ? [...t.componentsUsed] : [];
+ /** @type {string[]} */
+ let weakKeywords = [];
if (t.type === 'page') {
if (t.filePath) {
try {
- keywords = keywords.concat(extractComponents(t.filePath));
+ weakKeywords = extractComponents(t.filePath);
} catch {
// Best-effort: skip keyword enrichment if the source can't be read.
}
}
- if (t.category) keywords = keywords.concat(t.category.split(/[^A-Za-z0-9]+/).filter(Boolean));
+ if (t.category) keywords.push(...t.category.split(/[^A-Za-z0-9]+/).filter(Boolean));
}
return {
domain: 'template',
name: t.dirName,
keywords,
+ weakKeywords,
description: t.description || '',
_displayName: t.name,
_kind: t.type, // 'page' | 'block'
diff --git a/packages/cli/api/search/search.test.mjs b/packages/cli/api/search/search.test.mjs
index 5799d49611..951b6f38f7 100644
--- a/packages/cli/api/search/search.test.mjs
+++ b/packages/cli/api/search/search.test.mjs
@@ -23,7 +23,13 @@ import {describe, it, expect} from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import {fileURLToPath} from 'node:url';
-import {search, SEARCH_DOMAINS} from './search.mjs';
+import {
+ search,
+ scoreCandidate,
+ scoreQuery,
+ tokenizeQuery,
+ SEARCH_DOMAINS,
+} from './search.mjs';
const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..');
const cwd = REPO;
@@ -163,3 +169,75 @@ describe('search leaf — integration components', () => {
}
}, SLOW);
});
+
+describe('search scoring — multi-token aggregation is monotonic', () => {
+ /**
+ * @param {string} q
+ * @param {object} candidate
+ * @returns {number}
+ */
+ const score = (q, candidate) => scoreQuery(q, tokenizeQuery(q), candidate)?.score ?? 0;
+
+ it('ranks matching both terms above matching only the stronger one', () => {
+ // The shipped regression: scoring averaged over MATCHED tokens, so the
+ // weaker second hit pulled the mean down further than the coverage bonus
+ // pushed it up. `build "file browser"` put two form wizards that matched
+ // only "file" (98) above the actual file browser that matched both (97),
+ // and 98 clears the confident-match gate.
+ const partial = {name: 'form-wizard-vertical', keywords: ['file']};
+ const complete = {
+ name: 'file-explorer',
+ keywords: ['file'],
+ description: 'Column browser for nested folders',
+ };
+ expect(score('file browser', complete)).toBeGreaterThan(score('file browser', partial));
+ });
+
+ it('never scores a superset of matched terms below a subset', () => {
+ const subset = {name: 'zzz-none', keywords: ['alpha']};
+ const supersets = [
+ {name: 'zzz-none', keywords: ['alpha'], description: 'beta things'},
+ {name: 'zzz-none', keywords: ['alpha', 'beta']},
+ {name: 'zzz-none', keywords: ['alpha'], weakKeywords: ['beta']},
+ ];
+ for (const superset of supersets) {
+ expect(score('alpha beta', superset)).toBeGreaterThanOrEqual(score('alpha beta', subset));
+ }
+ });
+
+ it('still scores a verbose prompt on the concepts it did hit', () => {
+ // Guards the reason the mean was used: a long prompt matching one concept
+ // strongly must not be crushed by dividing across every query token.
+ const candidate = {name: 'kanban', keywords: ['kanban']};
+ expect(score('i need a kanban somewhere in this rambling request', candidate))
+ .toBeGreaterThanOrEqual(90);
+ });
+});
+
+describe('search scoring — derived keywords rank below authored ones', () => {
+ it('scores an authored keyword above a derived one', () => {
+ const authored = scoreCandidate('dialog', {name: 'x', keywords: ['Dialog']});
+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});
+ expect(authored?.score).toBe(90);
+ expect(derived?.score).toBe(60);
+ });
+
+ it('keeps one incidental derived match below the confident-match gate', () => {
+ // PAGE_DIRECT in api/build/kit is 95: at full keyword strength a page that
+ // renders one of everything got a 90-point shot per component and was
+ // returned as a confident match for queries it had nothing to do with.
+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});
+ expect(derived?.score).toBeLessThan(95);
+ });
+
+ it('still surfaces a derived match above the page floor', () => {
+ // PAGE_FLOOR is 50 — weakening the signal must not make it invisible.
+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});
+ expect(derived?.score).toBeGreaterThanOrEqual(50);
+ });
+
+ it('explains a derived hit as something the template renders', () => {
+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});
+ expect(derived?.reason).toMatch(/renders Dialog/);
+ });
+});
diff --git a/packages/cli/foundation/discovery/template-adapter.mjs b/packages/cli/foundation/discovery/template-adapter.mjs
index bd1d93285b..0fa1da19fb 100644
--- a/packages/cli/foundation/discovery/template-adapter.mjs
+++ b/packages/cli/foundation/discovery/template-adapter.mjs
@@ -672,7 +672,15 @@ export function extractComponents(pagePath) {
// (P2380608025), so the `XDS` prefix is optional. Anchoring on the `<`
// JSX-tag boundary keeps this precise (avoids matching imports/comments/
// identifiers) while remaining prefix-agnostic.
- const tagRegex = /<(XDS)?([A-Z]\w+)/g;
+ //
+ // The lookbehind additionally requires that the `<` NOT follow an identifier
+ // character, which is what separates a JSX tag from a TypeScript generic
+ // argument list. `return <Dialog` and `rows.map(r => <ListItem` match;
+ // `useState<ReadonlySet<string>>`, `ComponentType<SVGProps<SVGSVGElement>>`
+ // and `Record<string, Phase>` no longer do. Those were being indexed as
+ // rendered components, putting type names like `Record`, `SVGProps` and
+ // `ReadonlySet` into template keyword and "components used" output.
+ const tagRegex = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;
/** @type {string[]} */
const matches = [];
let m;
diff --git a/packages/cli/foundation/discovery/template-adapter.test.mjs b/packages/cli/foundation/discovery/template-adapter.test.mjs
new file mode 100644
index 0000000000..ecff0ba91a
--- /dev/null
+++ b/packages/cli/foundation/discovery/template-adapter.test.mjs
@@ -0,0 +1,136 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Colocated tests for template-adapter's `extractComponents`.
+ *
+ * `extractComponents` scrapes the components a page template renders out of its
+ * JSX. Three callers depend on it: `template <name>` and `template --skeleton`
+ * print it as "components used", and `search`/`build` index it as keywords.
+ * It had no test coverage, and the tag regex was matching TypeScript generic
+ * argument lists as if they were JSX tags — so type names (`Record`,
+ * `SVGProps`, `ReadonlySet`, and every local type alias) were being reported as
+ * rendered components and indexed as search keywords.
+ */
+
+import {describe, it, expect, beforeAll, afterAll} from 'vitest';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import {extractComponents} from './template-adapter.mjs';
+
+/** @type {string} */
+let dir;
+
+beforeAll(() => {
+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-extract-'));
+});
+
+afterAll(() => {
+ fs.rmSync(dir, {recursive: true, force: true});
+});
+
+/**
+ * @param {string} name
+ * @param {string} src
+ * @returns {string}
+ */
+function writePage(name, src) {
+ const file = path.join(dir, `${name}.tsx`);
+ fs.writeFileSync(file, src);
+ return file;
+}
+
+describe('extractComponents — JSX tags', () => {
+ it('extracts rendered components', () => {
+ const file = writePage(
+ 'basic',
+ `'use client';
+ export default function Page() {
+ return (
+ <Layout content={<Dialog><TreeList /></Dialog>} />
+ );
+ }`,
+ );
+ const got = extractComponents(file);
+ expect(got).toContain('Layout');
+ expect(got).toContain('Dialog');
+ expect(got).toContain('TreeList');
+ });
+
+ it('extracts a component opening a JSX expression, not just a statement', () => {
+ const file = writePage(
+ 'mapped',
+ `export default function Page() {
+ return <List>{rows.map(r => <ListRow key={r.id} />)}</List>;
+ }`,
+ );
+ expect(extractComponents(file)).toContain('ListRow');
+ });
+
+ it('drops ubiquitous primitives so they do not drown the signal', () => {
+ const file = writePage(
+ 'ubiquitous',
+ `export default function Page() {
+ return <VStack><Text>hi</Text><Button /><Kanban /></VStack>;
+ }`,
+ );
+ const got = extractComponents(file);
+ expect(got).not.toContain('VStack');
+ expect(got).not.toContain('Text');
+ expect(got).not.toContain('Button');
+ expect(got).toContain('Kanban');
+ });
+});
+
+describe('extractComponents — TypeScript generics are not components', () => {
+ it('ignores generic argument lists', () => {
+ const file = writePage(
+ 'generics',
+ `import type {ComponentType, SVGProps} from 'react';
+ type Phase = 'a' | 'b';
+ export default function Page() {
+ const [sel, setSel] = useState<ReadonlySet<string>>(new Set());
+ const rows = useMemo<Array<StockRow>>(() => [], []);
+ const icon: ComponentType<SVGProps<SVGSVGElement>> = Placeholder;
+ const byPhase: Record<string, Phase> = {};
+ return <Dialog />;
+ }`,
+ );
+ const got = extractComponents(file);
+ for (const typeName of [
+ 'ReadonlySet',
+ 'Array',
+ 'StockRow',
+ 'SVGProps',
+ 'SVGSVGElement',
+ 'Record',
+ 'Phase',
+ ]) {
+ expect(got).not.toContain(typeName);
+ }
+ expect(got).toContain('Dialog');
+ });
+
+ it('still ignores a generic when the JSX it precedes is valid', () => {
+ const file = writePage(
+ 'mixed',
+ `export default function Page() {
+ const m = new Map<string, Severity>();
+ return <Banner />;
+ }`,
+ );
+ const got = extractComponents(file);
+ expect(got).not.toContain('Severity');
+ expect(got).toEqual(['Banner']);
+ });
+
+ it('does not treat a closing tag as a component', () => {
+ const file = writePage(
+ 'closing',
+ `export default function Page() {
+ return <Card>x</Card>;
+ }`,
+ );
+ expect(extractComponents(file)).toEqual(['Card']);
+ });
+});