Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: the PR report reads the touched-components check's per-component results and wide-change flag, once #9178 writes them (#9217)

Cindy Zhang3ccdb9edcd1b
diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js
index f0201c2ec6..6764ab8b3c 100644
--- a/.github/scripts/generate-pr-comment.js
+++ b/.github/scripts/generate-pr-comment.js
@@ -3,7 +3,7 @@
/**
* @description Generates the PR comment from the run's analysis results
- * @input --analysis <file> --base-stats <file> --package-size <file> --a11y <file> --visual <file> --visual-frames <file> --preview <file> --jobs <file> --source-conclusion <conclusion> --run-url <url>
+ * @input --analysis <file> --base-stats <file> --package-size <file> --a11y <file> --visual <file> --visual-frames <file> --preview <file> --jobs <file> --touched <file> --source-conclusion <conclusion> --run-url <url>
* @output Formatted markdown comment body to stdout, compact by design: this
* head's preview first, the modified components (preview links, size, lines
* of code, a mark per browser suite that ran; three rows visible, the rest
@@ -60,11 +60,12 @@ if (baseStats) {
}
// This head's preview ({state, url, sha, aliasOrigin}), the source run's
-// jobs, the package-size report, and the
+// jobs, the touched-components report, the package-size report, and the
// browser suites' results. The accessibility report is absent when the
// audit did not run; the visual verdict when the visual job did not.
const preview = readOptionalJson('preview');
const jobs = readOptionalJson('jobs');
+const touched = readOptionalJson('touched');
const packageSize = readOptionalJson('package-size');
// The visual frames publish-frames.mjs put on this PR's assets branch.
const visualFrames = readOptionalJson('visual-frames');
@@ -124,6 +125,7 @@ const report = buildCompactReport({
analysis,
preview,
jobs,
+ touched,
a11yReport,
visualVerdict,
packageSize,
diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs
index 03fddfa6eb..79a878c431 100644
--- a/.github/scripts/lib/pr-preview.mjs
+++ b/.github/scripts/lib/pr-preview.mjs
@@ -618,6 +618,7 @@ export async function reconcilePrComment({
visualPath = 'trusted-visual/verdict.json',
createIfMissing = true,
fallbackMessage,
+ touchedPath = 'touched-components/touched-components.json',
generator = GENERATOR,
execute = execFileSync,
}) {
@@ -709,6 +710,14 @@ export async function reconcilePrComment({
...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []),
...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []),
...(framesPath ? ['--visual-frames', framesPath] : []),
+ ...(trustedHeadReport({
+ reportPath: touchedPath,
+ label: 'touched-components report',
+ identity,
+ core,
+ })
+ ? ['--touched', touchedPath]
+ : []),
...(preview ? ['--preview', previewPath] : []),
...(jobs ? ['--jobs', jobsPath] : []),
'--source-conclusion',
diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs
index 89d472245b..eb9c69bfd9 100644
--- a/.github/scripts/lib/pr-preview.test.mjs
+++ b/.github/scripts/lib/pr-preview.test.mjs
@@ -1133,6 +1133,24 @@ describe('PR comment preview reconciliation', () => {
},
);
+ it.each([
+ ['the exact head', HEAD, true],
+ ['another head', 'b'.repeat(40), false],
+ ])(
+ 'passes the touched-components report only when it names %s',
+ async (_label, headSha, used) => {
+ const seen = [];
+ await reconcile({
+ touched: {headSha, components: []},
+ execute: (_node, args) => {
+ seen.push(args);
+ return '## PR Analysis Report\n';
+ },
+ });
+ expect(seen[0].includes('--touched')).toBe(used);
+ },
+ );
+
it('falls back to a safe current message when analysis rendering fails', async () => {
const stale = {
id: 78,
diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js
index ba4c358b16..be63034bba 100644
--- a/.github/scripts/lib/pr-status.js
+++ b/.github/scripts/lib/pr-status.js
@@ -5,7 +5,7 @@
* @input The analysis (changed components, their import sizes and source
* stats at head and at the merge base), this head's Vercel preview state,
* the package-size report, the accessibility report, the visual verdict,
- * and the source CI run's jobs
+ * the source CI run's jobs and, when present, the touched-components report
* @output The PR report's compact body: this head's preview first; one table
* of modified components (preview links, JS and CSS size and lines of code
* against main, and a fixed-position mark per browser suite when any ran),
@@ -18,6 +18,7 @@
*/
const {inline, num, sizeText} = require('./report-text');
+const {SUITE_LABELS} = require('../run-labels.cjs');
/**
* The heavy suites a pull request runs only with its `ci:run-<suite>` label,
@@ -28,7 +29,9 @@ const SUITES = [
id: 'visual',
label: 'Visual',
short: 'Visual',
- jobs: /^(Stable visual|visual-scout|pr-visual)/i,
+ // The label-gated scout decides whether visual ran by label: the
+ // touched-components check may run the regression job without it.
+ jobs: /^(Stable visual scout|visual-scout)/i,
},
{
id: 'a11y',
@@ -37,7 +40,6 @@ const SUITES = [
jobs: /^(pr-a11y|Accessibility spec-test)/i,
},
{id: 'rtl', label: 'RTL', short: 'RTL', jobs: /^pr-rtl/i},
- {id: 'perf', label: 'Perf', short: 'Perf', jobs: /^(perf|pr-perf)/i},
];
/** Rows shown before the rest of a table is collapsed. */
@@ -167,14 +169,19 @@ function storybookLink(origin, storyTitle) {
/**
* One row per modified component: every component this change sized (its
* folder changed or its bundle reaches a changed module) and every new or
- * modified component folder. Results come from the accessibility report and
- * the visual verdict.
+ * modified component folder. Results come from the touched-components report
+ * when the run wrote one, otherwise from the accessibility report and the
+ * visual verdict.
*/
-function componentRows({analysis, a11yReport, visualVerdict}) {
+function componentRows({analysis, touched, a11yReport, visualVerdict}) {
const head = analysis.componentSizes || {};
const base = analysis.baseComponentSizes || {};
const stats = analysis.componentStats || {};
const baseStats = analysis.baseComponentStats || {};
+ // By key: Core and Lab can each have a component of one name.
+ const byKey = new Map(
+ (touched?.components || []).map(component => [component.key, component]),
+ );
const keys = new Set(Object.keys(head));
for (const name of [
...(analysis.newComponents || []),
@@ -186,6 +193,9 @@ function componentRows({analysis, a11yReport, visualVerdict}) {
);
keys.add(`${pkg}/${name}`);
}
+ for (const component of touched?.components || []) {
+ if (component.key) keys.add(component.key);
+ }
const visualChanges = visualVerdict?.changes || [];
// Main's sizes of the merge base were read (a base record or build).
const baseMeasured =
@@ -194,22 +204,34 @@ function componentRows({analysis, a11yReport, visualVerdict}) {
Object.keys(baseStats).length > 0;
return [...keys].map(key => {
const name = key.slice(key.indexOf('/') + 1);
+ // The touched-components check's own result comes first; `not-run` is no
+ // result, so the other reports still apply.
+ const reported = byKey.get(key)?.results || {};
+ const own = suite =>
+ reported[suite] && reported[suite] !== 'not-run' ? reported[suite] : null;
// A component the audit did not cover has no accessibility result.
const audited = a11yReport?.components?.[name];
- const a11y = audited
- ? (audited.violations?.length ?? 0) > 0
- ? 'fail'
- : 'pass'
- : null;
+ const a11y =
+ own('a11y') ??
+ (audited
+ ? (audited.violations?.length ?? 0) > 0
+ ? 'fail'
+ : 'pass'
+ : null);
+ // A verdict scoped to the touched components (#9178's run) compared only
+ // what the touched report lists; it is no result for any other row.
+ const verdictCovers =
+ visualVerdict && visualVerdict.context?.scope !== 'touched';
const visual =
- visualVerdict && visualVerdict.status !== 'skipped'
+ own('visual') ??
+ (verdictCovers && visualVerdict.status !== 'skipped'
? // A run that failed compared nothing: never a pass.
visualVerdict.status === 'failed'
? 'fail'
: visualChanges.some(entry => entry.component === name)
? 'changed'
: 'pass'
- : null;
+ : null);
return {
key,
name,
@@ -218,7 +240,7 @@ function componentRows({analysis, a11yReport, visualVerdict}) {
base: base[key],
growth: sizeChange(head[key], base[key]),
loc: [stats[name]?.linesOfCode, baseStats[name]?.linesOfCode],
- results: {visual, a11y, rtl: null},
+ results: {visual, a11y, rtl: own('rtl')},
baseMeasured,
};
});
@@ -242,15 +264,26 @@ function compareRows(a, b) {
return a.key.localeCompare(b.key);
}
-function buildComponentTable({analysis, origin, a11yReport, visualVerdict}) {
- const rows = componentRows({analysis, a11yReport, visualVerdict}).sort(
- compareRows,
- );
+function buildComponentTable({
+ analysis,
+ origin,
+ touched,
+ a11yReport,
+ visualVerdict,
+}) {
+ const rows = componentRows({
+ analysis,
+ touched,
+ a11yReport,
+ visualVerdict,
+ }).sort(compareRows);
if (rows.length === 0) return '';
- // The checks column appears only when a browser suite has a result here.
- const checks = rows.some(row =>
- CHECK_SUITES.some(id => row.results[id] != null),
- );
+ // The checks column appears when a browser suite has a result here, or
+ // when the change is too wide for the touched-components check (a suite
+ // run by its label still shows its marks).
+ const wide = touched?.wideChange === true;
+ const checks =
+ wide || rows.some(row => CHECK_SUITES.some(id => row.results[id] != null));
const columns = [
'Component',
'Size',
@@ -430,7 +463,7 @@ function packagesLine(report) {
* them. A failed or cancelled Storybook build skips them for another reason
* (GitHub shows that check), so the line is left out.
*/
-function notRunLine({jobs, a11yReport, visualVerdict}) {
+function notRunLine({jobs, a11yReport, visualVerdict, touched = null}) {
const build = (jobs || []).find(
job => baseJobName(job.name) === 'build-storybook',
);
@@ -444,11 +477,41 @@ function notRunLine({jobs, a11yReport, visualVerdict}) {
return false;
});
if (notRun.length === 0) return '';
- const labels = notRun.map(suite => `\`ci:run-${suite.id}\``);
+ const labels = notRun.map(suite => `\`${SUITE_LABELS[suite.id]}\``);
const list =
labels.length === 1
? labels[0]
: `${labels.slice(0, -1).join(', ')} or ${labels.at(-1)}`;
+ // Too wide for the touched-components check: it says so, and names only
+ // the suites that did not run. A narrower change's touched components were
+ // checked, so the labels run the full suites.
+ if (touched?.wideChange === true) {
+ // A shared-style change is wide whatever its component count; #9178
+ // records why in `foundations`.
+ const foundations = Array.isArray(touched.foundations)
+ ? touched.foundations.filter(Boolean)
+ : [];
+ const why =
+ foundations.length > 0
+ ? `Wide change (shared styles: ${foundations
+ .slice(0, 3)
+ .map(inline)
+ .join(
+ ', ',
+ )}${foundations.length > 3 ? `, and ${foundations.length - 3} more` : ''})`
+ : `Wide change (${num(touched.componentCount)} components)`;
+ return `${why}: add ${list} to run ${labels.length === 1 ? 'it' : 'them'}.\n\n`;
+ }
+ // Claim the touched check only when it produced a result.
+ const checked = (touched?.components || []).some(component =>
+ Object.values(component.results || {}).some(
+ result => result != null && result !== 'not-run',
+ ),
+ );
+ if (touched && checked)
+ return `Touched components checked; add ${list} to run the full suite${labels.length === 1 ? '' : 's'}.\n\n`;
+ if (touched)
+ return `Touched components identified; checks did not run. Add ${list} to run ${labels.length === 1 ? 'it' : 'them'}.\n\n`;
return `Not run on this PR: add ${list} to run ${labels.length === 1 ? 'it' : 'them'}.\n\n`;
}
@@ -457,6 +520,7 @@ function buildCompactReport({
analysis,
preview = null,
jobs = null,
+ touched = null,
a11yReport = null,
visualVerdict = null,
packageSize = null,
@@ -467,9 +531,10 @@ function buildCompactReport({
return `${previewLine(preview)}${buildComponentTable({
analysis,
origin,
+ touched,
a11yReport,
visualVerdict,
- })}${buildVisualDeltas({verdict: visualVerdict, frames: visualFrames})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict})}`;
+ })}${buildVisualDeltas({verdict: visualVerdict, frames: visualFrames})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`;
}
module.exports = {
diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs
index 636d9c0365..75009bca86 100644
--- a/.github/scripts/lib/pr-status.test.mjs
+++ b/.github/scripts/lib/pr-status.test.mjs
@@ -149,6 +149,98 @@ describe('buildComponentTable', () => {
expect(table).not.toContain('<details>');
});
+ // The touched-components check's report, as #9178 writes it.
+ const touchedReport = (components, extra = {}) => ({
+ headSha: HEAD,
+ wideChange: false,
+ componentCount: components.length,
+ components,
+ ...extra,
+ });
+
+ it('takes results from the touched-components report first, by key', () => {
+ const table = buildComponentTable({
+ analysis: analysis([
+ ['core/Chat', 1000, 1000],
+ ['lab/Chat', 900, 900],
+ ]),
+ origin: ORIGIN,
+ touched: touchedReport([
+ {
+ key: 'core/Chat',
+ package: 'core',
+ name: 'Chat',
+ results: {visual: 'changed', a11y: 'pass', rtl: 'fail'},
+ },
+ {
+ key: 'lab/Chat',
+ package: 'lab',
+ name: 'Chat',
+ results: {visual: 'pass', a11y: 'pass', rtl: 'pass'},
+ },
+ ]),
+ });
+ expect(table).toMatch(/^\| \[Chat\][^\n]* \| ✗ · ✓ · ✗ \|$/m);
+ expect(table).toMatch(/^\| \[lab\/Chat\][^\n]* \| ✓ · ✓ · ✓ \|$/m);
+ });
+
+ it('treats not-run as no result, so the other reports still apply', () => {
+ const table = buildComponentTable({
+ analysis: analysis([['core/A', 1000, 1000]]),
+ origin: ORIGIN,
+ touched: touchedReport([
+ {
+ key: 'core/A',
+ package: 'core',
+ name: 'A',
+ results: {visual: 'not-run', a11y: 'not-run', rtl: 'not-run'},
+ },
+ ]),
+ a11yReport: {components: {A: {violations: []}}},
+ });
+ expect(table).toMatch(/\| — · ✓ · — \|$/m);
+ });
+
+ it('never shows a visual pass from a touched-only verdict for a row the touched check did not compare', () => {
+ const table = buildComponentTable({
+ analysis: analysis([
+ ['core/New', 1000, 1000],
+ ['lab/Sketch', 900, 900],
+ ]),
+ origin: ORIGIN,
+ touched: touchedReport([
+ {
+ key: 'core/New',
+ package: 'core',
+ name: 'New',
+ results: {visual: 'not-run', a11y: 'pass', rtl: 'pass'},
+ },
+ ]),
+ visualVerdict: {status: 'ok', changes: [], context: {scope: 'touched'}},
+ });
+ expect(table).toMatch(/^\| \[New\][^\n]* \| — · ✓ · ✓ \|$/m);
+ expect(table).not.toMatch(/^\| \[lab\/Sketch\][^\n]* \| ✓/m);
+ });
+
+ it('keeps the marks of a suite run by its label on a wide change', () => {
+ const wide = {wideChange: true, componentCount: 60, components: []};
+ expect(
+ buildComponentTable({
+ analysis: analysis([['core/A', 1000, 1000]]),
+ origin: ORIGIN,
+ touched: wide,
+ }),
+ ).toMatch(/\| — · — · — \|$/m);
+ expect(
+ buildComponentTable({
+ analysis: analysis([['core/A', 1000, 1000]]),
+ origin: ORIGIN,
+ touched: wide,
+ a11yReport: {components: {A: {violations: []}}},
+ }),
+ ).toMatch(/\| — · ✓ · — \|$/m);
+ });
+
it('lists a modified component folder with no size', () => {
const table = buildComponentTable({
analysis: {
@@ -367,7 +459,7 @@ describe('notRunLine', () => {
notRunLine({
jobs: [
skipped('pr-rtl'),
- {name: 'Stable visual regression', conclusion: 'success'},
+ {name: 'Stable visual scout (1/5)', conclusion: 'success'},
{name: 'pr-a11y', conclusion: 'success'},
],
}),
@@ -390,6 +482,102 @@ describe('notRunLine', () => {
);
});
+ it('names only the suites that did not run on a wide change', () => {
+ const wide = {wideChange: true, componentCount: 60};
+ expect(
+ notRunLine({
+ jobs: [
+ skipped('Stable visual scout (1/2)'),
+ {name: 'pr-a11y (full audit)', conclusion: 'success'},
+ skipped('pr-rtl (core)'),
+ ],
+ touched: wide,
+ }),
+ ).toBe(
+ 'Wide change (60 components): add `ci:run-visual` or `ci:run-rtl` to run them.\n\n',
+ );
+ expect(notRunLine({jobs: [], touched: wide})).toBe('');
+ });
+
+ it('decides visual by its label-gated scout, not the regression job the touched check may run', () => {
+ expect(
+ notRunLine({
+ jobs: [
+ skipped('Stable visual scout (1/5)'),
+ {name: 'Stable visual regression', conclusion: 'success'},
+ {name: 'pr-a11y', conclusion: 'success'},
+ {name: 'pr-rtl', conclusion: 'success'},
+ ],
+ }),
+ ).toBe('Not run on this PR: add `ci:run-visual` to run it.\n\n');
+ });
+
+ it('says the touched components were checked on a narrow change', () => {
+ expect(
+ notRunLine({
+ jobs: [
+ skipped('Stable visual scout (1/5)'),
+ skipped('pr-a11y (full audit)'),
+ skipped('pr-rtl (core)'),
+ ],
+ touched: {
+ wideChange: false,
+ componentCount: 1,
+ components: [
+ {
+ key: 'core/A',
+ results: {visual: 'pass', a11y: 'not-run', rtl: 'not-run'},
+ },
+ ],
+ },
+ }),
+ ).toBe(
+ 'Touched components checked; add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run the full suites.\n\n',
+ );
+ // Every result not-run: never claim a check that did not happen.
+ expect(
+ notRunLine({
+ jobs: [
+ skipped('pr-rtl (core)'),
+ {name: 'Stable visual scout (1/5)', conclusion: 'success'},
+ {name: 'pr-a11y', conclusion: 'success'},
+ ],
+ touched: {
+ wideChange: false,
+ componentCount: 1,
+ components: [
+ {
+ key: 'core/A',
+ results: {visual: 'not-run', a11y: 'not-run', rtl: 'not-run'},
+ },
+ ],
+ },
+ }),
+ ).toBe(
+ 'Touched components identified; checks did not run. Add `ci:run-rtl` to run it.\n\n',
+ );
+ });
+
+ it('names the shared styles that made a change wide', () => {
+ expect(
+ notRunLine({
+ jobs: [
+ skipped('Stable visual scout (1/5)'),
+ skipped('pr-a11y (full audit)'),
+ skipped('pr-rtl (core)'),
+ ],
+ touched: {
+ wideChange: true,
+ componentCount: 0,
+ components: [],
+ foundations: ['packages/core/src/theme/tokens.stylex.ts'],
+ },
+ }),
+ ).toBe(
+ 'Wide change (shared styles: packages/core/src/theme/tokens.stylex.ts): add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run them.\n\n',
+ );
+ });
+
it('leaves the line out when the Storybook build failed', () => {
expect(
notRunLine({
diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml
index 2d7e81bce5..0edd6b8e3f 100644
--- a/.github/workflows/pr-comment.yml
+++ b/.github/workflows/pr-comment.yml
@@ -401,6 +401,18 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
continue-on-error: true
+ # The touched-components check's per-component results (or its wide-
+ # change flag); absent when the check did not run.
+ - name: Download the touched-components report from the CI run
+ if: steps.identity.outputs.valid == 'true'
+ uses: actions/download-artifact@v8
+ with:
+ name: touched-components
+ path: touched-components
+ run-id: ${{ github.event.workflow_run.id }}
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ continue-on-error: true
+
- name: Download a11y artifact from the CI run
if: steps.identity.outputs.valid == 'true'
uses: actions/download-artifact@v8