An open source design system that's fully customizable and agent ready
TypeScript
ci(pr-a11y): skip contract specs an older PR head does not have (#8240)
Christopher Chedeaua061fcbbf804
diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjsindex dcb4440537..3f98eef8df 100644--- a/.github/scripts/ci-test-routing.test.mjs+++ b/.github/scripts/ci-test-routing.test.mjs@@ -266,6 +266,77 @@ describe.each(Object.entries(WORKFLOWS))(},);+describe('ci.yml pr-a11y steps against an older PR head', () => {+ const steps = load('ci.yml').jobs['pr-a11y'].steps;+ /** The spec files each `pnpm exec playwright test` in a step names. */+ const specsOf = step =>+ (step.run ?? '')+ .replace(/\\\n/g, ' ')+ .split('\n')+ .filter(line => /pnpm exec playwright test\b/.test(line))+ .flatMap(line =>+ line+ .trim()+ .split(/\s+/)+ .filter(token => /\.spec\.(ts|tsx|js|mjs)$/.test(token)),+ );+ const playwright = steps.filter(step => specsOf(step).length > 0);++ // The wrapper skips spec files a PR head lacks, so a step that named a file+ // missing on main would pass silently everywhere. Main's steps must name+ // files that exist; only an older head skips anything.+ it('names only spec files that exist', () => {+ expect(playwright.length).toBeGreaterThan(0);+ for (const step of playwright) {+ for (const spec of specsOf(step)) {+ expect(+ fs.existsSync(path.join(root, spec)),+ `${step.name}: ${spec}`,+ ).toBe(true);+ }+ }+ });++ // A step that uploads its evidence with `if-no-files-found: error` must not+ // promise a run the wrapper then skips on an older head.+ it('promises heading-links evidence only when its spec is in the head', () => {+ const run = steps.find(+ step => step.name === 'Run heading-links module browser evidence',+ ).run;+ const spec =+ 'packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts';+ const promise = run.indexOf("echo 'should_run=true'");+ expect(promise).toBeGreaterThan(-1);+ expect(run.slice(0, promise)).toContain(`if [ -e ${spec} ]; then`);+ });++ it("sources the merge commit's absent-spec wrapper before every Playwright step", () => {+ const index = steps.findIndex(+ step => step.name === 'Skip contract specs the PR head predates',+ );+ expect(index).toBeGreaterThan(-1);+ const wrapper = steps[index];+ expect(wrapper.if).toBe("github.event_name == 'pull_request'");+ // The merge commit the steps come from, not the PR's recorded base, which+ // can lag main and predate the wrapper.+ expect(wrapper.env.WORKFLOW_SHA).toBe('${{ github.sha }}');+ expect(wrapper.env.BASE_SHA).toBeUndefined();+ expect(wrapper.run).toContain(+ 'git show "$WORKFLOW_SHA:.github/scripts/skip-absent-specs.sh" > "$wrapper"',+ );+ expect(wrapper.run).toContain('echo "BASH_ENV=$wrapper" >> "$GITHUB_ENV"');+ expect(+ fs.existsSync(path.join(root, '.github/scripts/skip-absent-specs.sh')),+ ).toBe(true);+ for (const step of playwright) {+ expect(steps.indexOf(step), step.name).toBeGreaterThan(index);+ // BASH_ENV reaches bash steps only, and only calls through the function.+ expect(step.shell ?? 'bash', step.name).toMatch(/^bash/);+ expect(step.run, step.name).not.toMatch(/\b(exec|command) pnpm\b/);+ }+ });+});+describe('ci.yml RTL package sharding', () => {const workflow = load('ci.yml');const shard = workflow.jobs['pr-rtl-shard'];diff --git a/.github/scripts/release-check-workflow.test.mjs b/.github/scripts/release-check-workflow.test.mjsindex 78073444f4..b83b387a81 100644--- a/.github/scripts/release-check-workflow.test.mjs+++ b/.github/scripts/release-check-workflow.test.mjs@@ -751,6 +751,11 @@ describe('full release scope without PR metadata or maintenance writes', () => {);expect(step('a11y-contracts', name).with['delete-merged']).toBe(true);}+ // An older PR head can lack a spec added on main; the exact main or+ // release head a release check runs on has every one.+ const wrapper = step('pr-a11y', 'Skip contract specs the PR head predates');+ expect(runs(wrapper, {event: 'pull_request', operation: ''})).toBe(true);+ expect(runs(wrapper)).toBe(false);for (const name of ['Run modal close visibility guard','Run theme var reachability guard',diff --git a/.github/scripts/skip-absent-specs.sh b/.github/scripts/skip-absent-specs.shnew file mode 100755index 0000000000..1b6e709bf4--- /dev/null+++ b/.github/scripts/skip-absent-specs.sh@@ -0,0 +1,49 @@+#!/usr/bin/env bash+# Copyright (c) Meta Platforms, Inc. and affiliates.+#+# Lets a pr-a11y step run against a PR head that predates its spec file.+#+# Not run directly: pr-a11y sources it into every later step through+# BASH_ENV. A PR step's body comes from the merge commit, but pr-a11y checks+# out the PR head. A contract step added on main after the PR branched names a+# spec file that head does not have, and Playwright fails the step with "No+# tests found".+#+# This wraps `pnpm exec playwright test`: spec files absent from the checkout+# are dropped from the command with a notice, and a step left with none passes+# without running Playwright. Every other pnpm command runs unchanged. The+# routing tests keep main's steps pointing at spec files that exist, so only an+# older head skips anything; rebasing it runs the new contract.+#+# The job loads this from the merge commit (github.sha) its steps come from,+# since an older PR head does not have it and the PR's recorded base can lag+# main.++pnpm() {+ if [ "${1-}" = exec ] && [ "${2-}" = playwright ] && [ "${3-}" = test ]; then+ local arg+ local kept=0+ local dropped=0+ local args=()+ for arg in "${@:4}"; do+ case "$arg" in+ -*) ;;+ *.spec.ts | *.spec.tsx | *.spec.js | *.spec.mjs)+ if [ ! -e "$arg" ]; then+ echo "::notice title=Spec not in this PR head::${arg} is not in this pull request's head; rebase onto main to run it."+ dropped=$((dropped + 1))+ continue+ fi+ kept=$((kept + 1))+ ;;+ esac+ args+=("$arg")+ done+ if [ "$dropped" -gt 0 ] && [ "$kept" -eq 0 ]; then+ echo "None of this step's spec files is in the PR head; nothing to run."+ return 0+ fi+ set -- exec playwright test "${args[@]}"+ fi+ command pnpm "$@"+}diff --git a/.github/scripts/skip-absent-specs.test.mjs b/.github/scripts/skip-absent-specs.test.mjsnew file mode 100644index 0000000000..f478b1bf85--- /dev/null+++ b/.github/scripts/skip-absent-specs.test.mjs@@ -0,0 +1,120 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file skip-absent-specs.test.mjs+ *+ * The wrapper pr-a11y sources into its steps so that a contract step added on+ * main does not fail an older PR head that lacks the step's spec file. Each+ * case runs a step script the way the job does (`bash -e`, the wrapper in+ * BASH_ENV) against a stand-in pnpm that records what it was asked to run.+ */++import {spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';++import {afterEach, beforeEach, describe, expect, it} from 'vitest';++const WRAPPER = path.join(import.meta.dirname, 'skip-absent-specs.sh');+const PRESENT = 'specs/Present.a11y.chromium.spec.ts';+const ABSENT = 'specs/Absent.a11y.chromium.spec.ts';++describe('skip-absent-specs.sh', () => {+ let dir;++ beforeEach(() => {+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-skip-absent-specs-'));+ fs.mkdirSync(path.join(dir, 'bin'));+ fs.writeFileSync(+ path.join(dir, 'bin', 'pnpm'),+ '#!/usr/bin/env bash\necho "$*" >> "$PNPM_LOG"\nexit "${PNPM_EXIT:-0}"\n',+ {mode: 0o755},+ );+ fs.mkdirSync(path.join(dir, 'specs'));+ fs.writeFileSync(path.join(dir, PRESENT), '');+ });++ afterEach(() => {+ fs.rmSync(dir, {recursive: true, force: true});+ });++ function step(script, env = {}) {+ const file = path.join(dir, 'step.sh');+ const log = path.join(dir, 'pnpm.log');+ fs.writeFileSync(file, script);+ const result = spawnSync('bash', ['-e', file], {+ cwd: dir,+ encoding: 'utf8',+ env: {+ ...process.env,+ BASH_ENV: WRAPPER,+ PATH: `${path.join(dir, 'bin')}${path.delimiter}${process.env.PATH}`,+ PNPM_LOG: log,+ ...env,+ },+ });+ const calls = fs.existsSync(log)+ ? fs.readFileSync(log, 'utf8').trim().split('\n')+ : [];+ return {status: result.status, stdout: result.stdout, calls};+ }++ it('runs a step whose spec files are all present unchanged', () => {+ const {status, stdout, calls} = step(+ `pnpm exec playwright test \\\n ${PRESENT} \\\n --project chromium \\\n --output out\n`,+ );+ expect(status).toBe(0);+ expect(calls).toEqual([+ `exec playwright test ${PRESENT} --project chromium --output out`,+ ]);+ expect(stdout).not.toContain('::notice');+ });++ it('drops a spec file the head lacks, with a notice, and runs the rest', () => {+ const {status, stdout, calls} = step(+ `pnpm exec playwright test ${ABSENT} ${PRESENT} --project chromium --reporter list,json --output out\n`,+ );+ expect(status).toBe(0);+ expect(calls).toEqual([+ `exec playwright test ${PRESENT} --project chromium --reporter list,json --output out`,+ ]);+ expect(stdout).toContain(+ `::notice title=Spec not in this PR head::${ABSENT} is not in this pull request's head; rebase onto main to run it.`,+ );+ });++ it('passes a step whose spec files are all absent without running Playwright', () => {+ const {status, stdout, calls} = step(+ `pnpm exec playwright test ${ABSENT} --project chromium --output out\necho after\n`,+ );+ expect(status).toBe(0);+ expect(calls).toEqual([]);+ expect(stdout).toContain(+ `::notice title=Spec not in this PR head::${ABSENT}`,+ );+ expect(stdout).toContain('nothing to run');+ expect(stdout).toContain('after');+ });++ it('keeps a failing Playwright run failing', () => {+ const {status, calls} = step(+ `pnpm exec playwright test ${PRESENT} --project chromium\necho unreachable\n`,+ {PNPM_EXIT: '1'},+ );+ expect(status).toBe(1);+ expect(calls).toHaveLength(1);+ });++ it('leaves every other pnpm command alone', () => {+ const {status, calls} = step(+ 'pnpm install --frozen-lockfile\npnpm exec playwright install chromium\npnpm exec vitest run x.spec.ts\n',+ );+ expect(status).toBe(0);+ expect(calls).toEqual([+ 'install --frozen-lockfile',+ 'exec playwright install chromium',+ 'exec vitest run x.spec.ts',+ ]);+ });+});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex fbc9b9ae6c..b452dcc1d3 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -1409,6 +1409,25 @@ jobs:EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA"+ # A contract step added on main after this PR branched names a spec file+ # the PR head does not have, and would fail with "No tests found". The+ # wrapper, sourced into every later step, drops such files from+ # `pnpm exec playwright test` with a notice and passes a step left with+ # none. It comes from the merge commit these steps come from (an older PR+ # head lacks it, and the PR's recorded base can lag main).+ - name: Skip contract specs the PR head predates+ if: github.event_name == 'pull_request'+ env:+ WORKFLOW_SHA: ${{ github.sha }}+ run: |+ wrapper="$RUNNER_TEMP/skip-absent-specs.sh"+ if git fetch --quiet --no-tags --depth=1 origin "$WORKFLOW_SHA" \+ && git show "$WORKFLOW_SHA:.github/scripts/skip-absent-specs.sh" > "$wrapper"; then+ echo "BASH_ENV=$wrapper" >> "$GITHUB_ENV"+ else+ echo "::warning::Could not load the spec wrapper from the merge commit; contract steps run unwrapped."+ fi+# Dependencies are installed later, together with the Playwright browser,# because the analysis artifact has to be downloaded first.- name: Setup Node and pnpm@@ -2653,7 +2672,13 @@ jobs:COMPONENTS=$(node .github/scripts/a11y-pr-scope.mjs analysis.json)case ",${COMPONENTS}," in*,core/Markdown,*)- echo 'should_run=true' >> "$GITHUB_OUTPUT"+ # An older PR head can lack the spec (the wrapper then skips+ # it), and it would leave nothing for the upload to find.+ if [ -e packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts ]; then+ echo 'should_run=true' >> "$GITHUB_OUTPUT"+ else+ echo 'should_run=false' >> "$GITHUB_OUTPUT"+ fipnpm exec playwright test \packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts \--project chromium \