An open source design system that's fully customizable and agent ready
TypeScript
fix(cli): stop build ranking partial query matches above full ones (#5614)
Ernestc6041cfd58ea
diff --git a/.changeset/build-retrieval-scoring.md b/.changeset/build-retrieval-scoring.mdnew file mode 100644index 0000000000..2921c8ec32--- /dev/null+++ b/.changeset/build-retrieval-scoring.md@@ -0,0 +1,7 @@+---+'@astryxdesign/cli': patch+---++[fix] `build`/`search` no longer rank a partially-matched template above one matching every term, no longer index TypeScript generic arguments as rendered components, and no longer treat breadth of rendered components as full-strength relevance.++@ernesttdiff --git a/packages/cli/api/search/search.mjs b/packages/cli/api/search/search.mjsindex 5091f7c350..4660f0ddf6 100644--- a/packages/cli/api/search/search.mjs+++ b/packages/cli/api/search/search.mjs@@ -21,13 +21,24 @@* 80 name Levenshtein distance 1* 70 keyword substring / distance 1* 60 name substring (>=4 chars, >=50% coverage)+ * 60 exact weak-keyword match* 50 description / prose mentions the term* 40 name Levenshtein distance 2+ * 40 weak-keyword substring* 30 keyword Levenshtein distance 2* 20 name Levenshtein distance 3** Name + keyword signals always outweigh description/prose, so an exact match* sorts above an incidental mention.+ *+ * `keywords` carry AUTHORED intent — a block's `componentsUsed`, a page's+ * `category` words. `weakKeywords` are DERIVED: the components a page template+ * happens to render, scraped out of its JSX. Derived signal is deliberately+ * capped below the confident-match gate on its own, because breadth is not+ * relevance. At full keyword strength every rendered component is an+ * independent 90-point shot with no penalty for how many a template has, so+ * the broadest pages (a theme showcase rendering one of everything) win+ * queries they have nothing to do with.*/import {pathToFileURL} from 'node:url';@@ -53,6 +64,7 @@ import {ERROR_CODES} from '../../foundation/response/error-codes.mjs';* @property {'component'|'hook'|'doc'|'template'} domain* @property {string} name* @property {string[]} [keywords]+ * @property {string[]} [weakKeywords]* @property {string} [description]* @property {string[]} [prose]* @property {string} [_import]@@ -222,28 +234,39 @@ export function scoreQuery(term, tokens, candidate) {// Multi-word natural language: score each content token, counting only// strong hits, then reward coverage so candidates matching more terms win.- let sum = 0;+ let strongest = 0;let matched = 0;/** @type {string[]} */const hitTerms = [];for (const tok of tokens) {const h = bestForToken(tok, candidate);if (h && h.score >= MIN_TOKEN_SCORE) {- sum += h.score;+ if (h.score > strongest) strongest = h.score;matched++;hitTerms.push(tok);}}if (matched === 0) return full;- // Reward the AVERAGE strength of the concepts that matched (not divided by- // total query length — that penalizes verbose / low-fidelity prompts), plus- // a bonus per additional matched concept and a coverage term. A candidate- // that matches several of the query's concepts beats one matching a single- // incidental word.- const avgMatched = sum / matched;+ // Base the score on the STRONGEST concept that matched, plus a bonus per+ // additional matched concept and a coverage term.+ //+ // Deliberately not the mean, and deliberately not divided by total query+ // length. Dividing by total length penalizes verbose / low-fidelity prompts.+ // Dividing by the number of MATCHED tokens (what this used to do) made the+ // score non-monotonic: a second, weaker hit could drag the mean down by more+ // than the coverage bonus added it back, so matching fewer terms well beat+ // matching more terms partially. Concretely, `build "file browser"` scored+ // two form wizards matching only "file" at 98, above the actual file browser+ // matching both terms at 97 — and 98 clears the PAGE_DIRECT gate, so the+ // wrong template was returned as a confident match.+ //+ // Taking the max keeps both properties: a verbose prompt is still scored on+ // the concepts it did hit, and matching a superset of another candidate's+ // terms can never score lower, since every term of the expression is+ // non-decreasing in the set of matched tokens.const coverage = matched / tokens.length;- const tokenScore = Math.round(avgMatched + Math.min(matched - 1, 3) * 12 + coverage * 15);+ const tokenScore = Math.round(strongest + Math.min(matched - 1, 3) * 12 + coverage * 15);if (full && full.score >= tokenScore) return full;return {@@ -260,12 +283,16 @@ export function scoreQuery(term, tokens, candidate) {* @param {string} term - Lowercased search term.* @param {object} candidate* @param {string} candidate.name - Primary identifier (component/hook name, topic, template name).- * @param {string[]} [candidate.keywords]+ * @param {string[]} [candidate.keywords] - Authored intent (componentsUsed, category words).+ * @param {string[]} [candidate.weakKeywords] - Derived signal (components a page renders).* @param {string} [candidate.description]* @param {string[]} [candidate.prose] - Extra free-text blobs (doc section text, best practices).* @returns {{score: number, reason: string} | null}*/-export function scoreCandidate(term, {name, keywords = [], description = '', prose = []}) {+export function scoreCandidate(+ term,+ {name, keywords = [], weakKeywords = [], description = '', prose = []},+) {let best = 0;let reason = '';/**@@ -314,6 +341,24 @@ export function scoreCandidate(term, {name, keywords = [], description = '', proelse if (dist === 2) consider(30, `keyword "${kw}" (distance ${dist})`);}+ // ── Weak keyword signals (derived, not authored) ─────────────────+ // Capped at 60 so one incidental component match cannot clear the+ // confident-match gate on its own; it still counts toward multi-term+ // coverage, which is where "this page renders that" is real evidence.+ // No Levenshtein tier — fuzzy matching a derived signal is pure noise.+ for (const kw of weakKeywords) {+ const kwLower = String(kw).toLowerCase();+ if (kwLower === term) {+ consider(60, `renders ${kw}`);+ continue;+ }+ const s = term.length < kwLower.length ? term : kwLower;+ const l = term.length < kwLower.length ? kwLower : term;+ if (s.length >= 4 && l.includes(s) && s.length / l.length >= 0.5) {+ consider(40, `renders ${kw}`);+ }+ }+// ── Prose / description signals (stem-tolerant whole word) ──────// Match the term's stem as a whole word, tolerating plural/gerund suffixes// so "chart" matches "charts" and "filter" matches "filtering".@@ -529,24 +574,33 @@ async function gatherTemplates(cwd) {return [];}return templates.map(t => {- // Blocks ship componentsUsed; page templates don't, so derive them from the- // source. Category words (e.g. "Dashboard - Analytics") are strong intent- // signal for pages, which otherwise only index on name + description.- let keywords = Array.isArray(t.componentsUsed) ? [...t.componentsUsed] : [];+ // Blocks ship an authored componentsUsed; page templates don't, so derive+ // them from the source. Category words (e.g. "Dashboard - Analytics") are+ // strong intent signal for pages, which otherwise only index on name ++ // description.+ //+ // Authored signal and derived signal are kept apart: componentsUsed and+ // category words are a deliberate statement of what the template is for,+ // while scraped JSX tags only say what it happens to render. See the+ // scoring table at the top of this file for why the derived set is capped.+ const keywords = Array.isArray(t.componentsUsed) ? [...t.componentsUsed] : [];+ /** @type {string[]} */+ let weakKeywords = [];if (t.type === 'page') {if (t.filePath) {try {- keywords = keywords.concat(extractComponents(t.filePath));+ weakKeywords = extractComponents(t.filePath);} catch {// Best-effort: skip keyword enrichment if the source can't be read.}}- if (t.category) keywords = keywords.concat(t.category.split(/[^A-Za-z0-9]+/).filter(Boolean));+ if (t.category) keywords.push(...t.category.split(/[^A-Za-z0-9]+/).filter(Boolean));}return {domain: 'template',name: t.dirName,keywords,+ weakKeywords,description: t.description || '',_displayName: t.name,_kind: t.type, // 'page' | 'block'diff --git a/packages/cli/api/search/search.test.mjs b/packages/cli/api/search/search.test.mjsindex 5799d49611..951b6f38f7 100644--- a/packages/cli/api/search/search.test.mjs+++ b/packages/cli/api/search/search.test.mjs@@ -23,7 +23,13 @@ import {describe, it, expect} from 'vitest';import * as fs from 'node:fs';import * as path from 'node:path';import {fileURLToPath} from 'node:url';-import {search, SEARCH_DOMAINS} from './search.mjs';+import {+ search,+ scoreCandidate,+ scoreQuery,+ tokenizeQuery,+ SEARCH_DOMAINS,+} from './search.mjs';const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../..');const cwd = REPO;@@ -163,3 +169,75 @@ describe('search leaf — integration components', () => {}}, SLOW);});++describe('search scoring — multi-token aggregation is monotonic', () => {+ /**+ * @param {string} q+ * @param {object} candidate+ * @returns {number}+ */+ const score = (q, candidate) => scoreQuery(q, tokenizeQuery(q), candidate)?.score ?? 0;++ it('ranks matching both terms above matching only the stronger one', () => {+ // The shipped regression: scoring averaged over MATCHED tokens, so the+ // weaker second hit pulled the mean down further than the coverage bonus+ // pushed it up. `build "file browser"` put two form wizards that matched+ // only "file" (98) above the actual file browser that matched both (97),+ // and 98 clears the confident-match gate.+ const partial = {name: 'form-wizard-vertical', keywords: ['file']};+ const complete = {+ name: 'file-explorer',+ keywords: ['file'],+ description: 'Column browser for nested folders',+ };+ expect(score('file browser', complete)).toBeGreaterThan(score('file browser', partial));+ });++ it('never scores a superset of matched terms below a subset', () => {+ const subset = {name: 'zzz-none', keywords: ['alpha']};+ const supersets = [+ {name: 'zzz-none', keywords: ['alpha'], description: 'beta things'},+ {name: 'zzz-none', keywords: ['alpha', 'beta']},+ {name: 'zzz-none', keywords: ['alpha'], weakKeywords: ['beta']},+ ];+ for (const superset of supersets) {+ expect(score('alpha beta', superset)).toBeGreaterThanOrEqual(score('alpha beta', subset));+ }+ });++ it('still scores a verbose prompt on the concepts it did hit', () => {+ // Guards the reason the mean was used: a long prompt matching one concept+ // strongly must not be crushed by dividing across every query token.+ const candidate = {name: 'kanban', keywords: ['kanban']};+ expect(score('i need a kanban somewhere in this rambling request', candidate))+ .toBeGreaterThanOrEqual(90);+ });+});++describe('search scoring — derived keywords rank below authored ones', () => {+ it('scores an authored keyword above a derived one', () => {+ const authored = scoreCandidate('dialog', {name: 'x', keywords: ['Dialog']});+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});+ expect(authored?.score).toBe(90);+ expect(derived?.score).toBe(60);+ });++ it('keeps one incidental derived match below the confident-match gate', () => {+ // PAGE_DIRECT in api/build/kit is 95: at full keyword strength a page that+ // renders one of everything got a 90-point shot per component and was+ // returned as a confident match for queries it had nothing to do with.+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});+ expect(derived?.score).toBeLessThan(95);+ });++ it('still surfaces a derived match above the page floor', () => {+ // PAGE_FLOOR is 50 — weakening the signal must not make it invisible.+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});+ expect(derived?.score).toBeGreaterThanOrEqual(50);+ });++ it('explains a derived hit as something the template renders', () => {+ const derived = scoreCandidate('dialog', {name: 'x', weakKeywords: ['Dialog']});+ expect(derived?.reason).toMatch(/renders Dialog/);+ });+});diff --git a/packages/cli/foundation/discovery/template-adapter.mjs b/packages/cli/foundation/discovery/template-adapter.mjsindex bd1d93285b..0fa1da19fb 100644--- a/packages/cli/foundation/discovery/template-adapter.mjs+++ b/packages/cli/foundation/discovery/template-adapter.mjs@@ -672,7 +672,15 @@ export function extractComponents(pagePath) {// (P2380608025), so the `XDS` prefix is optional. Anchoring on the `<`// JSX-tag boundary keeps this precise (avoids matching imports/comments/// identifiers) while remaining prefix-agnostic.- const tagRegex = /<(XDS)?([A-Z]\w+)/g;+ //+ // The lookbehind additionally requires that the `<` NOT follow an identifier+ // character, which is what separates a JSX tag from a TypeScript generic+ // argument list. `return <Dialog` and `rows.map(r => <ListItem` match;+ // `useState<ReadonlySet<string>>`, `ComponentType<SVGProps<SVGSVGElement>>`+ // and `Record<string, Phase>` no longer do. Those were being indexed as+ // rendered components, putting type names like `Record`, `SVGProps` and+ // `ReadonlySet` into template keyword and "components used" output.+ const tagRegex = /(?<![\w$.])<(XDS)?([A-Z]\w+)/g;/** @type {string[]} */const matches = [];let m;diff --git a/packages/cli/foundation/discovery/template-adapter.test.mjs b/packages/cli/foundation/discovery/template-adapter.test.mjsnew file mode 100644index 0000000000..ecff0ba91a--- /dev/null+++ b/packages/cli/foundation/discovery/template-adapter.test.mjs@@ -0,0 +1,136 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Colocated tests for template-adapter's `extractComponents`.+ *+ * `extractComponents` scrapes the components a page template renders out of its+ * JSX. Three callers depend on it: `template <name>` and `template --skeleton`+ * print it as "components used", and `search`/`build` index it as keywords.+ * It had no test coverage, and the tag regex was matching TypeScript generic+ * argument lists as if they were JSX tags — so type names (`Record`,+ * `SVGProps`, `ReadonlySet`, and every local type alias) were being reported as+ * rendered components and indexed as search keywords.+ */++import {describe, it, expect, beforeAll, afterAll} from 'vitest';+import * as fs from 'node:fs';+import * as os from 'node:os';+import * as path from 'node:path';+import {extractComponents} from './template-adapter.mjs';++/** @type {string} */+let dir;++beforeAll(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-extract-'));+});++afterAll(() => {+ fs.rmSync(dir, {recursive: true, force: true});+});++/**+ * @param {string} name+ * @param {string} src+ * @returns {string}+ */+function writePage(name, src) {+ const file = path.join(dir, `${name}.tsx`);+ fs.writeFileSync(file, src);+ return file;+}++describe('extractComponents — JSX tags', () => {+ it('extracts rendered components', () => {+ const file = writePage(+ 'basic',+ `'use client';+ export default function Page() {+ return (+ <Layout content={<Dialog><TreeList /></Dialog>} />+ );+ }`,+ );+ const got = extractComponents(file);+ expect(got).toContain('Layout');+ expect(got).toContain('Dialog');+ expect(got).toContain('TreeList');+ });++ it('extracts a component opening a JSX expression, not just a statement', () => {+ const file = writePage(+ 'mapped',+ `export default function Page() {+ return <List>{rows.map(r => <ListRow key={r.id} />)}</List>;+ }`,+ );+ expect(extractComponents(file)).toContain('ListRow');+ });++ it('drops ubiquitous primitives so they do not drown the signal', () => {+ const file = writePage(+ 'ubiquitous',+ `export default function Page() {+ return <VStack><Text>hi</Text><Button /><Kanban /></VStack>;+ }`,+ );+ const got = extractComponents(file);+ expect(got).not.toContain('VStack');+ expect(got).not.toContain('Text');+ expect(got).not.toContain('Button');+ expect(got).toContain('Kanban');+ });+});++describe('extractComponents — TypeScript generics are not components', () => {+ it('ignores generic argument lists', () => {+ const file = writePage(+ 'generics',+ `import type {ComponentType, SVGProps} from 'react';+ type Phase = 'a' | 'b';+ export default function Page() {+ const [sel, setSel] = useState<ReadonlySet<string>>(new Set());+ const rows = useMemo<Array<StockRow>>(() => [], []);+ const icon: ComponentType<SVGProps<SVGSVGElement>> = Placeholder;+ const byPhase: Record<string, Phase> = {};+ return <Dialog />;+ }`,+ );+ const got = extractComponents(file);+ for (const typeName of [+ 'ReadonlySet',+ 'Array',+ 'StockRow',+ 'SVGProps',+ 'SVGSVGElement',+ 'Record',+ 'Phase',+ ]) {+ expect(got).not.toContain(typeName);+ }+ expect(got).toContain('Dialog');+ });++ it('still ignores a generic when the JSX it precedes is valid', () => {+ const file = writePage(+ 'mixed',+ `export default function Page() {+ const m = new Map<string, Severity>();+ return <Banner />;+ }`,+ );+ const got = extractComponents(file);+ expect(got).not.toContain('Severity');+ expect(got).toEqual(['Banner']);+ });++ it('does not treat a closing tag as a component', () => {+ const file = writePage(+ 'closing',+ `export default function Page() {+ return <Card>x</Card>;+ }`,+ );+ expect(extractComponents(file)).toEqual(['Card']);+ });+});