Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

fix(ci): report PR Comment on the pull request, not on main (#7801)

Cindy Zhangfab4c0de6590
diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs
index 994777a2c2..c40b8b03b7 100644
--- a/.github/scripts/lib/pr-preview.test.mjs
+++ b/.github/scripts/lib/pr-preview.test.mjs
@@ -478,8 +478,14 @@ describe('trusted PR preview identity', () => {
'preview-comment',
'spec-only-reconcile',
'comment',
+ 'report',
]);
- for (const name of writeCapableJobs) {
+ // The report job writes only a commit status, for the head the trusted
+ // event names; it mutates no pull request.
+ expect(PR_COMMENT_WORKFLOW.jobs.report.permissions).toEqual({
+ statuses: 'write',
+ });
+ for (const name of writeCapableJobs.filter(job => job !== 'report')) {
const expected =
name === 'preview-comment'
? "needs.resolve-preview.outputs.valid == 'true'"
@@ -539,6 +545,119 @@ describe('trusted PR preview identity', () => {
});
});
+describe('PR Comment keeps its outcome off main', () => {
+ // workflow_run attaches these jobs to main's latest commit, so a failing step
+ // here would turn main red for a pull request's problem.
+ const LANE = ['resolve', 'spec-only-reconcile', 'comment', 'report'];
+ const OPTIONAL = new Set(['actions/download-artifact@v8']);
+
+ it.each(LANE)('never fails the run from the %s job', name => {
+ for (const step of PR_COMMENT_WORKFLOW.jobs[name].steps) {
+ expect(step['continue-on-error'], `${name}: ${step.name}`).toBe(true);
+ }
+ });
+
+ it.each(['resolve', 'spec-only-reconcile', 'comment'])(
+ 'counts every required %s step in the outcome it reports',
+ name => {
+ const job = PR_COMMENT_WORKFLOW.jobs[name];
+ for (const step of job.steps.filter(item => !OPTIONAL.has(item.uses))) {
+ expect(step.id, `${name}: ${step.name}`).toBeTruthy();
+ expect(job.outputs.failed, `${name}: ${step.name}`).toContain(
+ `steps.${step.id}.outcome == 'failure'`,
+ );
+ }
+ for (const step of job.steps.filter(item => OPTIONAL.has(item.uses))) {
+ expect(job.outputs.failed).not.toContain(`steps.${step.id}.`);
+ }
+ },
+ );
+
+ it('reports the outcome on the head of the run the event names', () => {
+ const report = PR_COMMENT_WORKFLOW.jobs.report;
+ expect(report.needs).toEqual(['resolve', 'spec-only-reconcile', 'comment']);
+ const [step] = report.steps;
+ expect(step.env.HEAD_SHA).toBe('${{ github.event.workflow_run.head_sha }}');
+ for (const name of ['resolve', 'spec-only-reconcile', 'comment']) {
+ expect(step.env.FAILED).toContain(
+ `needs.${name}.outputs.failed == 'true'`,
+ );
+ }
+ // A cancelled job published nothing; it is not an up-to-date report.
+ expect(step.env.FAILED).toContain("contains(needs.*.result, 'cancelled')");
+ });
+
+ it.each([
+ [
+ 'resolve',
+ 'Resolve trusted PR identity',
+ "steps.checkout.outcome == 'success'",
+ ],
+ [
+ 'spec-only-reconcile',
+ 'Remove stale preview links from an existing report',
+ "steps.checkout.outcome == 'success'",
+ ],
+ [
+ 'comment',
+ 'Reconfirm trusted PR identity',
+ "steps.setup.outcome == 'success'",
+ ],
+ ])(
+ 'still skips the %s work after its checkout or install fails',
+ (job, name, condition) => {
+ const step = PR_COMMENT_WORKFLOW.jobs[job].steps.find(
+ item => item.name === name,
+ );
+ expect(step.if).toBe(condition);
+ },
+ );
+
+ it('gates every later comment step on the confirmed identity', () => {
+ const steps = PR_COMMENT_WORKFLOW.jobs.comment.steps;
+ const after = steps.slice(
+ steps.findIndex(step => step.id === 'identity') + 1,
+ );
+ for (const step of after) {
+ expect(step.if, step.name).toContain(
+ "steps.identity.outputs.valid == 'true'",
+ );
+ }
+ });
+
+ it.each([
+ ['true', 'failure'],
+ ['false', 'success'],
+ ])('posts PR Comment as a status when FAILED=%s', async (failed, state) => {
+ const [step] = PR_COMMENT_WORKFLOW.jobs.report.steps;
+ const createCommitStatus = vi.fn(async () => ({}));
+ const previousEnv = {...process.env};
+ Object.assign(process.env, {HEAD_SHA: HEAD, FAILED: failed});
+ try {
+ await new ASYNC_FUNCTION('context', 'github', step.with.script)(
+ {
+ repo: {owner: 'facebook', repo: 'astryx'},
+ serverUrl: 'https://github.com',
+ runId: 42,
+ },
+ {rest: {repos: {createCommitStatus}}},
+ );
+ } finally {
+ process.env = previousEnv;
+ }
+ expect(createCommitStatus).toHaveBeenCalledWith(
+ expect.objectContaining({
+ owner: 'facebook',
+ repo: 'astryx',
+ sha: HEAD,
+ context: 'PR Comment',
+ state,
+ target_url: 'https://github.com/facebook/astryx/actions/runs/42',
+ }),
+ );
+ });
+});
+
describe('confirmSourceRunIdentity reads the attempt the event reported', () => {
it('accepts the reported attempt after a newer rerun of the same run started', async () => {
const expected = identity({
diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md
index 899daa51bd..592cb4a68b 100644
--- a/.github/scripts/visual-gate/README.md
+++ b/.github/scripts/visual-gate/README.md
@@ -77,7 +77,8 @@ come from an earlier attempt of that run, never a later one.
`pr-comment.yml` only publishes its report: it checks identity, bounds JSON and
image sizes, re-encodes PNGs, and renders escaped HTML from default-branch code.
It does not execute artifact HTML, run a browser, compare pixels again, change
-a baseline, or create another visual status.
+a baseline, or create another visual status. Its only status is `PR Comment` on
+the pull request head, which says whether a report step failed.
Deployment previews, `a11y-weekly.yml`, `rtl-weekly.yml`, and
`vibe-screenshots.yml` remain separate because they do not run visual regression.
diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
index ee654b4d78..ed2c969861 100644
--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs
+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
@@ -39,7 +39,17 @@ describe('PR report and deployment workflow contracts', () => {
expect(value).toContain('types: [completed]');
expect(value).not.toMatch(/gate\.mjs (?:check|capture|release)/);
expect(value).not.toContain('playwright');
- expect(value).not.toContain('createCommitStatus');
+ // Its one status says whether the report was published, on the pull
+ // request head; it carries no visual verdict.
+ const parsed = yaml.parse(value);
+ for (const [name, job] of Object.entries(parsed.jobs)) {
+ if (name === 'report') continue;
+ expect(JSON.stringify(job), name).not.toContain('createCommitStatus');
+ }
+ const reportScript = parsed.jobs.report.steps[0].with.script;
+ expect(reportScript.match(/createCommitStatus/g)).toHaveLength(1);
+ expect(reportScript).toContain("context: 'PR Comment'");
+ expect(reportScript).not.toMatch(/visual|verdict/i);
});
it('serializes early preview and later CI enrichment per PR, not across PRs', () => {
diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml
index 7a74d44955..532a200ccf 100644
--- a/.github/workflows/pr-comment.yml
+++ b/.github/workflows/pr-comment.yml
@@ -45,14 +45,21 @@ jobs:
source_conclusion: ${{ steps.identity.outputs.source_conclusion }}
spec_only: ${{ steps.identity.outputs.spec_only }}
tooling_only: ${{ steps.identity.outputs.tooling_only }}
+ failed: ${{ steps.checkout.outcome == 'failure' || steps.identity.outcome == 'failure' }}
steps:
- name: Checkout trusted default-branch code
+ id: checkout
+ continue-on-error: true
uses: actions/checkout@v7
with:
ref: main
- name: Resolve trusted PR identity
+ continue-on-error: true
id: identity
+ # Steps continue on error, so a failed checkout no longer skips this
+ # one by itself.
+ if: steps.checkout.outcome == 'success'
uses: actions/github-script@v9
with:
retries: 3
@@ -218,16 +225,25 @@ jobs:
actions: read
contents: read
pull-requests: write
+ outputs:
+ failed: ${{ steps.scope.outcome == 'failure' || steps.checkout.outcome == 'failure' || steps.reconcile.outcome == 'failure' }}
steps:
- name: Report no visual scope
+ id: scope
+ continue-on-error: true
run: echo 'No visual scope applies; canonical visual checks are not applicable.' >> "$GITHUB_STEP_SUMMARY"
- name: Checkout trusted default-branch code
+ id: checkout
+ continue-on-error: true
uses: actions/checkout@v7
with:
ref: main
- name: Remove stale preview links from an existing report
+ id: reconcile
+ continue-on-error: true
+ if: steps.checkout.outcome == 'success'
uses: actions/github-script@v9
env:
PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -286,19 +302,37 @@ jobs:
pull-requests: write
actions: read
deployments: read # resolve the exact-head Vercel Preview deployment
+ # The artifact downloads are optional (a run may not produce them), so
+ # they are not failures of this job.
+ outputs:
+ failed: >-
+ ${{ steps.checkout.outcome == 'failure' ||
+ steps.setup.outcome == 'failure' ||
+ steps.identity.outcome == 'failure' ||
+ steps.cross-check.outcome == 'failure' ||
+ steps.visual.outcome == 'failure' ||
+ steps.publish.outcome == 'failure' }}
steps:
- name: Checkout trusted default-branch code
+ id: checkout
+ continue-on-error: true
uses: actions/checkout@v7
with:
ref: main
- name: Setup Node and pnpm
+ id: setup
+ continue-on-error: true
uses: ./.github/actions/setup
# Recheck the exact source run and current PR immediately before any
# privileged report or status mutation.
- name: Reconfirm trusted PR identity
+ continue-on-error: true
id: identity
+ # Steps continue on error, so a failed checkout or install no longer
+ # skips the rest by itself. Every later step requires a valid identity.
+ if: steps.setup.outcome == 'success'
uses: actions/github-script@v9
env:
PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}
@@ -381,6 +415,8 @@ jobs:
continue-on-error: true
- name: Cross-check artifact identity
+ id: cross-check
+ continue-on-error: true
if: steps.identity.outputs.valid == 'true' && hashFiles('pr-analysis/pr-meta.json') != ''
env:
EXPECTED_PR: ${{ steps.identity.outputs.pr_number }}
@@ -407,6 +443,8 @@ jobs:
# Only JSON and re-encoded PNGs from the exact CI attempt are published.
# No browser, baseline comparison, acceptance, or status authority lives here.
- name: Prepare canonical visual report
+ id: visual
+ continue-on-error: true
if: steps.identity.outputs.valid == 'true' && steps.visual-download.outcome == 'success'
env:
HEAD_SHA: ${{ steps.identity.outputs.head_sha }}
@@ -421,6 +459,8 @@ jobs:
--run-attempt "$RUN_ATTEMPT"
- name: Generate and post PR comment
+ id: publish
+ continue-on-error: true
if: always() && steps.identity.outputs.valid == 'true'
uses: actions/github-script@v9
env:
@@ -458,3 +498,49 @@ jobs:
sourceConclusion: process.env.SOURCE_CONCLUSION,
},
});
+
+ # The CI-completion lane runs on workflow_run, so GitHub attaches its jobs to
+ # main's latest commit. Main's checks must show main's health, not a pull
+ # request's. The jobs above therefore never fail the run: every fallible step
+ # continues on error, and this job reports their outcome as the PR Comment
+ # status on the pull request's own head. (The deployment_status lane already
+ # runs on the pull request's head commit.)
+ report:
+ needs: [resolve, spec-only-reconcile, comment]
+ if: >-
+ always() &&
+ github.event_name == 'workflow_run' &&
+ github.event.workflow_run.event == 'pull_request' &&
+ (needs.resolve.outputs.valid == 'true' ||
+ needs.resolve.outputs.failed == 'true')
+ runs-on: ubuntu-slim
+ permissions:
+ statuses: write
+ steps:
+ - name: Report the outcome on the pull request head
+ continue-on-error: true
+ uses: actions/github-script@v9
+ env:
+ # The source run's head is the pull request head it tested, taken from
+ # the trusted event rather than from any job output.
+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+ FAILED: >-
+ ${{ needs.resolve.outputs.failed == 'true' ||
+ needs.spec-only-reconcile.outputs.failed == 'true' ||
+ needs.comment.outputs.failed == 'true' ||
+ contains(needs.*.result, 'failure') ||
+ contains(needs.*.result, 'cancelled') }}
+ with:
+ retries: 3
+ script: |
+ const failed = process.env.FAILED === 'true';
+ await github.rest.repos.createCommitStatus({
+ ...context.repo,
+ sha: process.env.HEAD_SHA,
+ context: 'PR Comment',
+ state: failed ? 'failure' : 'success',
+ target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ description: failed
+ ? 'A PR report step failed or was cancelled. See the run.'
+ : 'No PR report step failed.',
+ });