Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci(pr-a11y): skip contract specs an older PR head does not have (#8240)

Christopher Chedeaua061fcbbf804
diff --git a/.github/scripts/ci-test-routing.test.mjs b/.github/scripts/ci-test-routing.test.mjs
index dcb4440537..3f98eef8df 100644
--- a/.github/scripts/ci-test-routing.test.mjs
+++ b/.github/scripts/ci-test-routing.test.mjs
@@ -266,6 +266,77 @@ describe.each(Object.entries(WORKFLOWS))(
},
);
+describe('ci.yml pr-a11y steps against an older PR head', () => {
+ const steps = load('ci.yml').jobs['pr-a11y'].steps;
+ /** The spec files each `pnpm exec playwright test` in a step names. */
+ const specsOf = step =>
+ (step.run ?? '')
+ .replace(/\\\n/g, ' ')
+ .split('\n')
+ .filter(line => /pnpm exec playwright test\b/.test(line))
+ .flatMap(line =>
+ line
+ .trim()
+ .split(/\s+/)
+ .filter(token => /\.spec\.(ts|tsx|js|mjs)$/.test(token)),
+ );
+ const playwright = steps.filter(step => specsOf(step).length > 0);
+
+ // The wrapper skips spec files a PR head lacks, so a step that named a file
+ // missing on main would pass silently everywhere. Main's steps must name
+ // files that exist; only an older head skips anything.
+ it('names only spec files that exist', () => {
+ expect(playwright.length).toBeGreaterThan(0);
+ for (const step of playwright) {
+ for (const spec of specsOf(step)) {
+ expect(
+ fs.existsSync(path.join(root, spec)),
+ `${step.name}: ${spec}`,
+ ).toBe(true);
+ }
+ }
+ });
+
+ // A step that uploads its evidence with `if-no-files-found: error` must not
+ // promise a run the wrapper then skips on an older head.
+ it('promises heading-links evidence only when its spec is in the head', () => {
+ const run = steps.find(
+ step => step.name === 'Run heading-links module browser evidence',
+ ).run;
+ const spec =
+ 'packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts';
+ const promise = run.indexOf("echo 'should_run=true'");
+ expect(promise).toBeGreaterThan(-1);
+ expect(run.slice(0, promise)).toContain(`if [ -e ${spec} ]; then`);
+ });
+
+ it("sources the merge commit's absent-spec wrapper before every Playwright step", () => {
+ const index = steps.findIndex(
+ step => step.name === 'Skip contract specs the PR head predates',
+ );
+ expect(index).toBeGreaterThan(-1);
+ const wrapper = steps[index];
+ expect(wrapper.if).toBe("github.event_name == 'pull_request'");
+ // The merge commit the steps come from, not the PR's recorded base, which
+ // can lag main and predate the wrapper.
+ expect(wrapper.env.WORKFLOW_SHA).toBe('${{ github.sha }}');
+ expect(wrapper.env.BASE_SHA).toBeUndefined();
+ expect(wrapper.run).toContain(
+ 'git show "$WORKFLOW_SHA:.github/scripts/skip-absent-specs.sh" > "$wrapper"',
+ );
+ expect(wrapper.run).toContain('echo "BASH_ENV=$wrapper" >> "$GITHUB_ENV"');
+ expect(
+ fs.existsSync(path.join(root, '.github/scripts/skip-absent-specs.sh')),
+ ).toBe(true);
+ for (const step of playwright) {
+ expect(steps.indexOf(step), step.name).toBeGreaterThan(index);
+ // BASH_ENV reaches bash steps only, and only calls through the function.
+ expect(step.shell ?? 'bash', step.name).toMatch(/^bash/);
+ expect(step.run, step.name).not.toMatch(/\b(exec|command) pnpm\b/);
+ }
+ });
+});
+
describe('ci.yml RTL package sharding', () => {
const workflow = load('ci.yml');
const shard = workflow.jobs['pr-rtl-shard'];
diff --git a/.github/scripts/release-check-workflow.test.mjs b/.github/scripts/release-check-workflow.test.mjs
index 78073444f4..b83b387a81 100644
--- a/.github/scripts/release-check-workflow.test.mjs
+++ b/.github/scripts/release-check-workflow.test.mjs
@@ -751,6 +751,11 @@ describe('full release scope without PR metadata or maintenance writes', () => {
);
expect(step('a11y-contracts', name).with['delete-merged']).toBe(true);
}
+ // An older PR head can lack a spec added on main; the exact main or
+ // release head a release check runs on has every one.
+ const wrapper = step('pr-a11y', 'Skip contract specs the PR head predates');
+ expect(runs(wrapper, {event: 'pull_request', operation: ''})).toBe(true);
+ expect(runs(wrapper)).toBe(false);
for (const name of [
'Run modal close visibility guard',
'Run theme var reachability guard',
diff --git a/.github/scripts/skip-absent-specs.sh b/.github/scripts/skip-absent-specs.sh
new file mode 100755
index 0000000000..1b6e709bf4
--- /dev/null
+++ b/.github/scripts/skip-absent-specs.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Lets a pr-a11y step run against a PR head that predates its spec file.
+#
+# Not run directly: pr-a11y sources it into every later step through
+# BASH_ENV. A PR step's body comes from the merge commit, but pr-a11y checks
+# out the PR head. A contract step added on main after the PR branched names a
+# spec file that head does not have, and Playwright fails the step with "No
+# tests found".
+#
+# This wraps `pnpm exec playwright test`: spec files absent from the checkout
+# are dropped from the command with a notice, and a step left with none passes
+# without running Playwright. Every other pnpm command runs unchanged. The
+# routing tests keep main's steps pointing at spec files that exist, so only an
+# older head skips anything; rebasing it runs the new contract.
+#
+# The job loads this from the merge commit (github.sha) its steps come from,
+# since an older PR head does not have it and the PR's recorded base can lag
+# main.
+
+pnpm() {
+ if [ "${1-}" = exec ] && [ "${2-}" = playwright ] && [ "${3-}" = test ]; then
+ local arg
+ local kept=0
+ local dropped=0
+ local args=()
+ for arg in "${@:4}"; do
+ case "$arg" in
+ -*) ;;
+ *.spec.ts | *.spec.tsx | *.spec.js | *.spec.mjs)
+ if [ ! -e "$arg" ]; then
+ echo "::notice title=Spec not in this PR head::${arg} is not in this pull request's head; rebase onto main to run it."
+ dropped=$((dropped + 1))
+ continue
+ fi
+ kept=$((kept + 1))
+ ;;
+ esac
+ args+=("$arg")
+ done
+ if [ "$dropped" -gt 0 ] && [ "$kept" -eq 0 ]; then
+ echo "None of this step's spec files is in the PR head; nothing to run."
+ return 0
+ fi
+ set -- exec playwright test "${args[@]}"
+ fi
+ command pnpm "$@"
+}
diff --git a/.github/scripts/skip-absent-specs.test.mjs b/.github/scripts/skip-absent-specs.test.mjs
new file mode 100644
index 0000000000..f478b1bf85
--- /dev/null
+++ b/.github/scripts/skip-absent-specs.test.mjs
@@ -0,0 +1,120 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file skip-absent-specs.test.mjs
+ *
+ * The wrapper pr-a11y sources into its steps so that a contract step added on
+ * main does not fail an older PR head that lacks the step's spec file. Each
+ * case runs a step script the way the job does (`bash -e`, the wrapper in
+ * BASH_ENV) against a stand-in pnpm that records what it was asked to run.
+ */
+
+import {spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+
+const WRAPPER = path.join(import.meta.dirname, 'skip-absent-specs.sh');
+const PRESENT = 'specs/Present.a11y.chromium.spec.ts';
+const ABSENT = 'specs/Absent.a11y.chromium.spec.ts';
+
+describe('skip-absent-specs.sh', () => {
+ let dir;
+
+ beforeEach(() => {
+ dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-skip-absent-specs-'));
+ fs.mkdirSync(path.join(dir, 'bin'));
+ fs.writeFileSync(
+ path.join(dir, 'bin', 'pnpm'),
+ '#!/usr/bin/env bash\necho "$*" >> "$PNPM_LOG"\nexit "${PNPM_EXIT:-0}"\n',
+ {mode: 0o755},
+ );
+ fs.mkdirSync(path.join(dir, 'specs'));
+ fs.writeFileSync(path.join(dir, PRESENT), '');
+ });
+
+ afterEach(() => {
+ fs.rmSync(dir, {recursive: true, force: true});
+ });
+
+ function step(script, env = {}) {
+ const file = path.join(dir, 'step.sh');
+ const log = path.join(dir, 'pnpm.log');
+ fs.writeFileSync(file, script);
+ const result = spawnSync('bash', ['-e', file], {
+ cwd: dir,
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ BASH_ENV: WRAPPER,
+ PATH: `${path.join(dir, 'bin')}${path.delimiter}${process.env.PATH}`,
+ PNPM_LOG: log,
+ ...env,
+ },
+ });
+ const calls = fs.existsSync(log)
+ ? fs.readFileSync(log, 'utf8').trim().split('\n')
+ : [];
+ return {status: result.status, stdout: result.stdout, calls};
+ }
+
+ it('runs a step whose spec files are all present unchanged', () => {
+ const {status, stdout, calls} = step(
+ `pnpm exec playwright test \\\n ${PRESENT} \\\n --project chromium \\\n --output out\n`,
+ );
+ expect(status).toBe(0);
+ expect(calls).toEqual([
+ `exec playwright test ${PRESENT} --project chromium --output out`,
+ ]);
+ expect(stdout).not.toContain('::notice');
+ });
+
+ it('drops a spec file the head lacks, with a notice, and runs the rest', () => {
+ const {status, stdout, calls} = step(
+ `pnpm exec playwright test ${ABSENT} ${PRESENT} --project chromium --reporter list,json --output out\n`,
+ );
+ expect(status).toBe(0);
+ expect(calls).toEqual([
+ `exec playwright test ${PRESENT} --project chromium --reporter list,json --output out`,
+ ]);
+ expect(stdout).toContain(
+ `::notice title=Spec not in this PR head::${ABSENT} is not in this pull request's head; rebase onto main to run it.`,
+ );
+ });
+
+ it('passes a step whose spec files are all absent without running Playwright', () => {
+ const {status, stdout, calls} = step(
+ `pnpm exec playwright test ${ABSENT} --project chromium --output out\necho after\n`,
+ );
+ expect(status).toBe(0);
+ expect(calls).toEqual([]);
+ expect(stdout).toContain(
+ `::notice title=Spec not in this PR head::${ABSENT}`,
+ );
+ expect(stdout).toContain('nothing to run');
+ expect(stdout).toContain('after');
+ });
+
+ it('keeps a failing Playwright run failing', () => {
+ const {status, calls} = step(
+ `pnpm exec playwright test ${PRESENT} --project chromium\necho unreachable\n`,
+ {PNPM_EXIT: '1'},
+ );
+ expect(status).toBe(1);
+ expect(calls).toHaveLength(1);
+ });
+
+ it('leaves every other pnpm command alone', () => {
+ const {status, calls} = step(
+ 'pnpm install --frozen-lockfile\npnpm exec playwright install chromium\npnpm exec vitest run x.spec.ts\n',
+ );
+ expect(status).toBe(0);
+ expect(calls).toEqual([
+ 'install --frozen-lockfile',
+ 'exec playwright install chromium',
+ 'exec vitest run x.spec.ts',
+ ]);
+ });
+});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fbc9b9ae6c..b452dcc1d3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1409,6 +1409,25 @@ jobs:
EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD_SHA"
+ # A contract step added on main after this PR branched names a spec file
+ # the PR head does not have, and would fail with "No tests found". The
+ # wrapper, sourced into every later step, drops such files from
+ # `pnpm exec playwright test` with a notice and passes a step left with
+ # none. It comes from the merge commit these steps come from (an older PR
+ # head lacks it, and the PR's recorded base can lag main).
+ - name: Skip contract specs the PR head predates
+ if: github.event_name == 'pull_request'
+ env:
+ WORKFLOW_SHA: ${{ github.sha }}
+ run: |
+ wrapper="$RUNNER_TEMP/skip-absent-specs.sh"
+ if git fetch --quiet --no-tags --depth=1 origin "$WORKFLOW_SHA" \
+ && git show "$WORKFLOW_SHA:.github/scripts/skip-absent-specs.sh" > "$wrapper"; then
+ echo "BASH_ENV=$wrapper" >> "$GITHUB_ENV"
+ else
+ echo "::warning::Could not load the spec wrapper from the merge commit; contract steps run unwrapped."
+ fi
+
# Dependencies are installed later, together with the Playwright browser,
# because the analysis artifact has to be downloaded first.
- name: Setup Node and pnpm
@@ -2653,7 +2672,13 @@ jobs:
COMPONENTS=$(node .github/scripts/a11y-pr-scope.mjs analysis.json)
case ",${COMPONENTS}," in
*,core/Markdown,*)
- echo 'should_run=true' >> "$GITHUB_OUTPUT"
+ # An older PR head can lack the spec (the wrapper then skips
+ # it), and it would leave nothing for the upload to find.
+ if [ -e packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts ]; then
+ echo 'should_run=true' >> "$GITHUB_OUTPUT"
+ else
+ echo 'should_run=false' >> "$GITHUB_OUTPUT"
+ fi
pnpm exec playwright test \
packages/core/src/Markdown/plugins/headingLinks.a11y.chromium.spec.ts \
--project chromium \