An open source design system that's fully customizable and agent ready
TypeScript
fix(ci): report PR Comment on the pull request, not on main (#7801)
Cindy Zhangfab4c0de6590
diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjsindex 994777a2c2..c40b8b03b7 100644--- a/.github/scripts/lib/pr-preview.test.mjs+++ b/.github/scripts/lib/pr-preview.test.mjs@@ -478,8 +478,14 @@ describe('trusted PR preview identity', () => {'preview-comment','spec-only-reconcile','comment',+ 'report',]);- for (const name of writeCapableJobs) {+ // The report job writes only a commit status, for the head the trusted+ // event names; it mutates no pull request.+ expect(PR_COMMENT_WORKFLOW.jobs.report.permissions).toEqual({+ statuses: 'write',+ });+ for (const name of writeCapableJobs.filter(job => job !== 'report')) {const expected =name === 'preview-comment'? "needs.resolve-preview.outputs.valid == 'true'"@@ -539,6 +545,119 @@ describe('trusted PR preview identity', () => {});});+describe('PR Comment keeps its outcome off main', () => {+ // workflow_run attaches these jobs to main's latest commit, so a failing step+ // here would turn main red for a pull request's problem.+ const LANE = ['resolve', 'spec-only-reconcile', 'comment', 'report'];+ const OPTIONAL = new Set(['actions/download-artifact@v8']);++ it.each(LANE)('never fails the run from the %s job', name => {+ for (const step of PR_COMMENT_WORKFLOW.jobs[name].steps) {+ expect(step['continue-on-error'], `${name}: ${step.name}`).toBe(true);+ }+ });++ it.each(['resolve', 'spec-only-reconcile', 'comment'])(+ 'counts every required %s step in the outcome it reports',+ name => {+ const job = PR_COMMENT_WORKFLOW.jobs[name];+ for (const step of job.steps.filter(item => !OPTIONAL.has(item.uses))) {+ expect(step.id, `${name}: ${step.name}`).toBeTruthy();+ expect(job.outputs.failed, `${name}: ${step.name}`).toContain(+ `steps.${step.id}.outcome == 'failure'`,+ );+ }+ for (const step of job.steps.filter(item => OPTIONAL.has(item.uses))) {+ expect(job.outputs.failed).not.toContain(`steps.${step.id}.`);+ }+ },+ );++ it('reports the outcome on the head of the run the event names', () => {+ const report = PR_COMMENT_WORKFLOW.jobs.report;+ expect(report.needs).toEqual(['resolve', 'spec-only-reconcile', 'comment']);+ const [step] = report.steps;+ expect(step.env.HEAD_SHA).toBe('${{ github.event.workflow_run.head_sha }}');+ for (const name of ['resolve', 'spec-only-reconcile', 'comment']) {+ expect(step.env.FAILED).toContain(+ `needs.${name}.outputs.failed == 'true'`,+ );+ }+ // A cancelled job published nothing; it is not an up-to-date report.+ expect(step.env.FAILED).toContain("contains(needs.*.result, 'cancelled')");+ });++ it.each([+ [+ 'resolve',+ 'Resolve trusted PR identity',+ "steps.checkout.outcome == 'success'",+ ],+ [+ 'spec-only-reconcile',+ 'Remove stale preview links from an existing report',+ "steps.checkout.outcome == 'success'",+ ],+ [+ 'comment',+ 'Reconfirm trusted PR identity',+ "steps.setup.outcome == 'success'",+ ],+ ])(+ 'still skips the %s work after its checkout or install fails',+ (job, name, condition) => {+ const step = PR_COMMENT_WORKFLOW.jobs[job].steps.find(+ item => item.name === name,+ );+ expect(step.if).toBe(condition);+ },+ );++ it('gates every later comment step on the confirmed identity', () => {+ const steps = PR_COMMENT_WORKFLOW.jobs.comment.steps;+ const after = steps.slice(+ steps.findIndex(step => step.id === 'identity') + 1,+ );+ for (const step of after) {+ expect(step.if, step.name).toContain(+ "steps.identity.outputs.valid == 'true'",+ );+ }+ });++ it.each([+ ['true', 'failure'],+ ['false', 'success'],+ ])('posts PR Comment as a status when FAILED=%s', async (failed, state) => {+ const [step] = PR_COMMENT_WORKFLOW.jobs.report.steps;+ const createCommitStatus = vi.fn(async () => ({}));+ const previousEnv = {...process.env};+ Object.assign(process.env, {HEAD_SHA: HEAD, FAILED: failed});+ try {+ await new ASYNC_FUNCTION('context', 'github', step.with.script)(+ {+ repo: {owner: 'facebook', repo: 'astryx'},+ serverUrl: 'https://github.com',+ runId: 42,+ },+ {rest: {repos: {createCommitStatus}}},+ );+ } finally {+ process.env = previousEnv;+ }+ expect(createCommitStatus).toHaveBeenCalledWith(+ expect.objectContaining({+ owner: 'facebook',+ repo: 'astryx',+ sha: HEAD,+ context: 'PR Comment',+ state,+ target_url: 'https://github.com/facebook/astryx/actions/runs/42',+ }),+ );+ });+});+describe('confirmSourceRunIdentity reads the attempt the event reported', () => {it('accepts the reported attempt after a newer rerun of the same run started', async () => {const expected = identity({diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.mdindex 899daa51bd..592cb4a68b 100644--- a/.github/scripts/visual-gate/README.md+++ b/.github/scripts/visual-gate/README.md@@ -77,7 +77,8 @@ come from an earlier attempt of that run, never a later one.`pr-comment.yml` only publishes its report: it checks identity, bounds JSON andimage sizes, re-encodes PNGs, and renders escaped HTML from default-branch code.It does not execute artifact HTML, run a browser, compare pixels again, change-a baseline, or create another visual status.+a baseline, or create another visual status. Its only status is `PR Comment` on+the pull request head, which says whether a report step failed.Deployment previews, `a11y-weekly.yml`, `rtl-weekly.yml`, and`vibe-screenshots.yml` remain separate because they do not run visual regression.diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjsindex ee654b4d78..ed2c969861 100644--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs@@ -39,7 +39,17 @@ describe('PR report and deployment workflow contracts', () => {expect(value).toContain('types: [completed]');expect(value).not.toMatch(/gate\.mjs (?:check|capture|release)/);expect(value).not.toContain('playwright');- expect(value).not.toContain('createCommitStatus');+ // Its one status says whether the report was published, on the pull+ // request head; it carries no visual verdict.+ const parsed = yaml.parse(value);+ for (const [name, job] of Object.entries(parsed.jobs)) {+ if (name === 'report') continue;+ expect(JSON.stringify(job), name).not.toContain('createCommitStatus');+ }+ const reportScript = parsed.jobs.report.steps[0].with.script;+ expect(reportScript.match(/createCommitStatus/g)).toHaveLength(1);+ expect(reportScript).toContain("context: 'PR Comment'");+ expect(reportScript).not.toMatch(/visual|verdict/i);});it('serializes early preview and later CI enrichment per PR, not across PRs', () => {diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.ymlindex 7a74d44955..532a200ccf 100644--- a/.github/workflows/pr-comment.yml+++ b/.github/workflows/pr-comment.yml@@ -45,14 +45,21 @@ jobs:source_conclusion: ${{ steps.identity.outputs.source_conclusion }}spec_only: ${{ steps.identity.outputs.spec_only }}tooling_only: ${{ steps.identity.outputs.tooling_only }}+ failed: ${{ steps.checkout.outcome == 'failure' || steps.identity.outcome == 'failure' }}steps:- name: Checkout trusted default-branch code+ id: checkout+ continue-on-error: trueuses: actions/checkout@v7with:ref: main- name: Resolve trusted PR identity+ continue-on-error: trueid: identity+ # Steps continue on error, so a failed checkout no longer skips this+ # one by itself.+ if: steps.checkout.outcome == 'success'uses: actions/github-script@v9with:retries: 3@@ -218,16 +225,25 @@ jobs:actions: readcontents: readpull-requests: write+ outputs:+ failed: ${{ steps.scope.outcome == 'failure' || steps.checkout.outcome == 'failure' || steps.reconcile.outcome == 'failure' }}steps:- name: Report no visual scope+ id: scope+ continue-on-error: truerun: echo 'No visual scope applies; canonical visual checks are not applicable.' >> "$GITHUB_STEP_SUMMARY"- name: Checkout trusted default-branch code+ id: checkout+ continue-on-error: trueuses: actions/checkout@v7with:ref: main- name: Remove stale preview links from an existing report+ id: reconcile+ continue-on-error: true+ if: steps.checkout.outcome == 'success'uses: actions/github-script@v9env:PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}@@ -286,19 +302,37 @@ jobs:pull-requests: writeactions: readdeployments: read # resolve the exact-head Vercel Preview deployment+ # The artifact downloads are optional (a run may not produce them), so+ # they are not failures of this job.+ outputs:+ failed: >-+ ${{ steps.checkout.outcome == 'failure' ||+ steps.setup.outcome == 'failure' ||+ steps.identity.outcome == 'failure' ||+ steps.cross-check.outcome == 'failure' ||+ steps.visual.outcome == 'failure' ||+ steps.publish.outcome == 'failure' }}steps:- name: Checkout trusted default-branch code+ id: checkout+ continue-on-error: trueuses: actions/checkout@v7with:ref: main- name: Setup Node and pnpm+ id: setup+ continue-on-error: trueuses: ./.github/actions/setup# Recheck the exact source run and current PR immediately before any# privileged report or status mutation.- name: Reconfirm trusted PR identity+ continue-on-error: trueid: identity+ # Steps continue on error, so a failed checkout or install no longer+ # skips the rest by itself. Every later step requires a valid identity.+ if: steps.setup.outcome == 'success'uses: actions/github-script@v9env:PR_NUMBER: ${{ needs.resolve.outputs.pr_number }}@@ -381,6 +415,8 @@ jobs:continue-on-error: true- name: Cross-check artifact identity+ id: cross-check+ continue-on-error: trueif: steps.identity.outputs.valid == 'true' && hashFiles('pr-analysis/pr-meta.json') != ''env:EXPECTED_PR: ${{ steps.identity.outputs.pr_number }}@@ -407,6 +443,8 @@ jobs:# Only JSON and re-encoded PNGs from the exact CI attempt are published.# No browser, baseline comparison, acceptance, or status authority lives here.- name: Prepare canonical visual report+ id: visual+ continue-on-error: trueif: steps.identity.outputs.valid == 'true' && steps.visual-download.outcome == 'success'env:HEAD_SHA: ${{ steps.identity.outputs.head_sha }}@@ -421,6 +459,8 @@ jobs:--run-attempt "$RUN_ATTEMPT"- name: Generate and post PR comment+ id: publish+ continue-on-error: trueif: always() && steps.identity.outputs.valid == 'true'uses: actions/github-script@v9env:@@ -458,3 +498,49 @@ jobs:sourceConclusion: process.env.SOURCE_CONCLUSION,},});++ # The CI-completion lane runs on workflow_run, so GitHub attaches its jobs to+ # main's latest commit. Main's checks must show main's health, not a pull+ # request's. The jobs above therefore never fail the run: every fallible step+ # continues on error, and this job reports their outcome as the PR Comment+ # status on the pull request's own head. (The deployment_status lane already+ # runs on the pull request's head commit.)+ report:+ needs: [resolve, spec-only-reconcile, comment]+ if: >-+ always() &&+ github.event_name == 'workflow_run' &&+ github.event.workflow_run.event == 'pull_request' &&+ (needs.resolve.outputs.valid == 'true' ||+ needs.resolve.outputs.failed == 'true')+ runs-on: ubuntu-slim+ permissions:+ statuses: write+ steps:+ - name: Report the outcome on the pull request head+ continue-on-error: true+ uses: actions/github-script@v9+ env:+ # The source run's head is the pull request head it tested, taken from+ # the trusted event rather than from any job output.+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}+ FAILED: >-+ ${{ needs.resolve.outputs.failed == 'true' ||+ needs.spec-only-reconcile.outputs.failed == 'true' ||+ needs.comment.outputs.failed == 'true' ||+ contains(needs.*.result, 'failure') ||+ contains(needs.*.result, 'cancelled') }}+ with:+ retries: 3+ script: |+ const failed = process.env.FAILED === 'true';+ await github.rest.repos.createCommitStatus({+ ...context.repo,+ sha: process.env.HEAD_SHA,+ context: 'PR Comment',+ state: failed ? 'failure' : 'success',+ target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,+ description: failed+ ? 'A PR report step failed or was cancelled. See the run.'+ : 'No PR report step failed.',+ });