Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: run the CDN page render's browser cases, and hold every browser-gated script test to a CI step (#10021)

Cindy Zhangd8387c5a9afe
diff --git a/.github/scripts/ci-browser-gated-tests.test.mjs b/.github/scripts/ci-browser-gated-tests.test.mjs
new file mode 100644
index 0000000000..4b7f31951e
--- /dev/null
+++ b/.github/scripts/ci-browser-gated-tests.test.mjs
@@ -0,0 +1,302 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file Contract for the steps that run browser-gated script tests in CI.
+ *
+ * Test runs that install no browser skip a test whose browser cases run only
+ * behind a switch (`ASTRYX_<NAME>_BROWSER=1`) or only where Chromium is
+ * installed. Each such test file under .github/scripts needs a workflow step
+ * that runs it after a Chromium install in the same job, under that install's
+ * condition or a narrower one that adds to it, with the file's switch set to
+ * '1' so a missing browser fails instead of skipping.
+ */
+
+import {spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+import {describe, expect, it} from 'vitest';
+import yaml from 'yaml';
+
+const root = path.resolve(import.meta.dirname, '../..');
+const WORKFLOWS = Object.fromEntries(
+ fs
+ .readdirSync(path.join(root, '.github/workflows'))
+ .filter(name => /\.ya?ml$/.test(name))
+ .map(name => [
+ name,
+ yaml.parse(
+ fs.readFileSync(path.join(root, '.github/workflows', name), 'utf8'),
+ ),
+ ]),
+);
+const OFFLINE_MEDIA_TEST = '.github/scripts/lib/offline-media.test.mjs';
+const CDN_BROWSER_TEST = '.github/scripts/lib/cdn-page-render.browser.test.mjs';
+const RTL_D7_TEST = '.github/scripts/rtl-audit-coarse-hit.browser.test.mjs';
+const INSTALLS_CHROMIUM = /\bplaywright install(?: --with-deps)? chromium\b/;
+const BROWSER_SWITCH = /process\.env\.(ASTRYX_[A-Z_]+_BROWSER) === '1'/;
+// A suite declared at the start of a line, so a fixture string in a test
+// does not count.
+const RUN_IF_SUITE = /^describe\.runIf\(/m;
+
+/** Whether a test file's browser cases run only behind a switch or an installed Chromium. */
+function browserGated(source) {
+ return (
+ RUN_IF_SUITE.test(source) &&
+ (BROWSER_SWITCH.test(source) || /\bchromiumInstalled\b/.test(source))
+ );
+}
+
+function scriptTests(dir) {
+ return fs
+ .readdirSync(path.join(root, dir), {withFileTypes: true})
+ .flatMap(entry => {
+ const rel = path.posix.join(dir, entry.name);
+ if (entry.isDirectory()) return scriptTests(rel);
+ return entry.name.endsWith('.test.mjs') ? [rel] : [];
+ });
+}
+
+/** Every step that runs `file`, with its workflow, job, the job's steps and its index. */
+function stepsRunning(file, workflows) {
+ const found = [];
+ for (const [workflow, parsed] of Object.entries(workflows)) {
+ for (const [jobName, job] of Object.entries(parsed?.jobs ?? {})) {
+ (job.steps ?? []).forEach((step, index) => {
+ if (typeof step.run === 'string' && step.run.includes(file)) {
+ found.push({workflow, jobName, steps: job.steps, index, step});
+ }
+ });
+ }
+ }
+ return found;
+}
+
+/** Whether one pair of parentheses wraps the whole expression. */
+function wrapped(expression) {
+ if (!expression.startsWith('(') || !expression.endsWith(')')) return false;
+ let depth = 0;
+ for (let i = 0; i < expression.length; i++) {
+ if (expression[i] === '(') depth++;
+ else if (expression[i] === ')') depth--;
+ if (depth === 0 && i < expression.length - 1) return false;
+ }
+ return depth === 0;
+}
+
+/**
+ * The install's condition, or a narrower one that ANDs a parenthesised part
+ * onto it. Unparenthesised, `X || Y && <install>` reads as `X || (Y && ...)`,
+ * which runs without the install whenever X holds.
+ */
+function sameOrNarrower(stepIf, installIf) {
+ if (installIf == null) return true;
+ if (stepIf === installIf) return true;
+ if (typeof stepIf !== 'string') return false;
+ const safeInstall = wrapped(installIf) || !installIf.includes('||');
+ const tail = ` && ${installIf}`;
+ const head = `${installIf} && `;
+ if (stepIf.endsWith(tail)) {
+ return safeInstall && wrapped(stepIf.slice(0, -tail.length));
+ }
+ if (stepIf.startsWith(head)) {
+ return safeInstall && wrapped(stepIf.slice(head.length));
+ }
+ return false;
+}
+
+/** Problems with how the workflows run one browser-gated test file. */
+function browserStepProblems(file, source, workflows) {
+ const browserSwitch = BROWSER_SWITCH.exec(source)?.[1];
+ if (browserSwitch == null) {
+ return [`${file} has no ASTRYX_*_BROWSER switch to require its browser`];
+ }
+ const runs = stepsRunning(file, workflows).filter(
+ ({step}) => step.env?.[browserSwitch] === '1',
+ );
+ if (runs.length === 0) {
+ return [`no workflow step runs ${file} with ${browserSwitch}=1`];
+ }
+ const problems = [];
+ for (const {workflow, jobName, steps, index, step} of runs) {
+ const where = `${workflow} ${jobName}: "${step.name}"`;
+ const install = steps
+ .slice(0, index)
+ .findLast(earlier => INSTALLS_CHROMIUM.test(earlier.run ?? ''));
+ if (install == null) {
+ problems.push(`${where} runs before any Chromium install`);
+ } else if (!sameOrNarrower(step.if, install.if)) {
+ problems.push(
+ `${where} runs under a condition that is not "${install.name}"'s or narrower`,
+ );
+ }
+ }
+ return problems;
+}
+
+describe('browser-gated script tests in the workflows', () => {
+ const gated = scriptTests('.github/scripts').filter(file =>
+ browserGated(fs.readFileSync(path.join(root, file), 'utf8')),
+ );
+
+ it('finds every browser-gated script test', () => {
+ expect(gated).toEqual(
+ expect.arrayContaining([
+ OFFLINE_MEDIA_TEST,
+ CDN_BROWSER_TEST,
+ RTL_D7_TEST,
+ ]),
+ );
+ });
+
+ it.each(gated)(
+ 'runs %s after a Chromium install, under its condition, with its browser switch',
+ file => {
+ const source = fs.readFileSync(path.join(root, file), 'utf8');
+ expect(browserStepProblems(file, source, WORKFLOWS)).toEqual([]);
+ },
+ );
+
+ it('runs the offline-media cases in the Node test job, with the browser required', () => {
+ const [run, ...more] = stepsRunning(OFFLINE_MEDIA_TEST, WORKFLOWS);
+ expect(more).toEqual([]);
+ expect([run.workflow, run.jobName]).toEqual(['ci.yml', 'test-node']);
+ expect(run.step.name).toBe('Offline-media browser cases');
+ expect(run.step.env).toEqual({ASTRYX_OFFLINE_MEDIA_BROWSER: '1'});
+ expect(run.step.run).toBe(
+ `pnpm exec vitest run ${OFFLINE_MEDIA_TEST} --maxWorkers=1`,
+ );
+ });
+
+ it('runs the CDN page render cases in the CLI smoke test, beside the CDN smoke test', () => {
+ const [run, ...more] = stepsRunning(CDN_BROWSER_TEST, WORKFLOWS);
+ expect(more).toEqual([]);
+ expect([run.workflow, run.jobName]).toEqual([
+ 'cli-smoke-test.yml',
+ 'smoke-test',
+ ]);
+ expect(run.step.env).toEqual({ASTRYX_CDN_SMOKE_BROWSER: '1'});
+ expect(run.step.run).toBe(
+ `pnpm exec vitest run --config .github/scripts/vitest.browser-scripts.config.mjs ${CDN_BROWSER_TEST} --maxWorkers=1`,
+ );
+ // The workflow starts for a change to the page renderer, its tests or
+ // the config the step runs (spec:AST-030 FR14).
+ const paths = WORKFLOWS['cli-smoke-test.yml'].on.pull_request.paths;
+ expect(paths).toContain('.github/scripts/lib/cdn-*.mjs');
+ expect(paths).toContain(
+ '.github/scripts/vitest.browser-scripts.config.mjs',
+ );
+ });
+
+ it('runs the CDN file under a config with no global setup, which still finds it', async () => {
+ const {default: config} =
+ await import('./vitest.browser-scripts.config.mjs');
+ expect(config.test.globalSetup).toBeUndefined();
+ expect(config.test.projects).toBeUndefined();
+ expect(
+ config.test.include.some(glob =>
+ path.matchesGlob(CDN_BROWSER_TEST, glob),
+ ),
+ ).toBe(true);
+ });
+
+ it('keeps the full suite, and so the install, for a change to offline-media', () => {
+ for (const file of [
+ '.github/scripts/lib/offline-media.js',
+ OFFLINE_MEDIA_TEST,
+ ]) {
+ const result = spawnSync(
+ 'node',
+ [path.join(root, '.github/scripts/change-scope.cjs')],
+ {input: `M\t${file}\n`, encoding: 'utf8'},
+ );
+ expect({file, ...JSON.parse(result.stdout)}).toMatchObject({
+ file,
+ docsiteOnly: false,
+ specOnly: false,
+ toolingOnly: false,
+ fullSuite: true,
+ });
+ }
+ });
+
+ describe('the check itself', () => {
+ const source =
+ "describe.runIf(process.env.ASTRYX_X_BROWSER === '1' || chromiumInstalled)";
+ const install = {
+ name: 'Install',
+ if: 'a',
+ run: 'npx playwright install chromium',
+ };
+ const runStep = (over = {}) => ({
+ name: 'Run',
+ if: 'a',
+ env: {ASTRYX_X_BROWSER: '1'},
+ run: 'pnpm exec vitest run x.test.mjs',
+ ...over,
+ });
+ const check = steps =>
+ browserStepProblems('x.test.mjs', source, {
+ 'w.yml': {jobs: {job: {steps}}},
+ });
+
+ it('accepts a run after the install under the same condition, or a narrower one', () => {
+ expect(check([install, runStep()])).toEqual([]);
+ expect(check([install, runStep({if: '(b) && a'})])).toEqual([]);
+ });
+
+ it.each([
+ [
+ 'a run before the install',
+ [runStep(), install],
+ 'runs before any Chromium install',
+ ],
+ ['a different condition', [install, runStep({if: 'b'})], 'not "Install"'],
+ [
+ 'a wider condition',
+ [install, runStep({if: 'a || b'})],
+ 'not "Install"',
+ ],
+ [
+ 'an unparenthesised narrower part',
+ [install, runStep({if: 'x || y && a'})],
+ 'not "Install"',
+ ],
+ [
+ 'no browser switch on the step',
+ [install, runStep({env: {}})],
+ 'no workflow step runs x.test.mjs',
+ ],
+ ['no step at all', [install], 'no workflow step runs x.test.mjs'],
+ ])('flags %s', (_, steps, problem) => {
+ expect(check(steps).join('\n')).toContain(problem);
+ });
+
+ it('flags a gated file with no browser switch', () => {
+ expect(
+ browserStepProblems(
+ 'x.test.mjs',
+ 'describe.runIf(chromiumInstalled)',
+ {},
+ ),
+ ).toEqual([
+ 'x.test.mjs has no ASTRYX_*_BROWSER switch to require its browser',
+ ]);
+ });
+
+ it('counts a suite gated on a switch or on an installed Chromium, not a fixture string', () => {
+ expect(
+ browserGated("describe.runIf(process.env.ASTRYX_Y_BROWSER === '1')("),
+ ).toBe(true);
+ expect(
+ browserGated('const ok = chromiumInstalled;\ndescribe.runIf(ok)('),
+ ).toBe(true);
+ expect(
+ browserGated(
+ 'const s = "describe.runIf(process.env.ASTRYX_Y_BROWSER === \'1\')";',
+ ),
+ ).toBe(false);
+ expect(browserGated("describe('x', () => {});")).toBe(false);
+ });
+ });
+});
diff --git a/.github/scripts/ci-offline-media-browser.test.mjs b/.github/scripts/ci-offline-media-browser.test.mjs
deleted file mode 100644
index 73f9203e66..0000000000
--- a/.github/scripts/ci-offline-media-browser.test.mjs
+++ /dev/null
@@ -1,198 +0,0 @@
-// Copyright (c) Meta Platforms, Inc. and affiliates.
-
-/**
- * @file Contract for the step that runs browser-gated script tests in CI.
- *
- * The Node test legs run before any browser is installed, so a test that
- * runs only where Chromium is installed skips there. Each such test file
- * needs a ci.yml step that runs it after a Chromium install, under that
- * install's own condition, with the file's browser switch set to '1' so a
- * missing browser fails instead of skipping.
- */
-
-import {spawnSync} from 'node:child_process';
-import fs from 'node:fs';
-import path from 'node:path';
-
-import {describe, expect, it} from 'vitest';
-import yaml from 'yaml';
-
-const root = path.resolve(import.meta.dirname, '../..');
-const ci = yaml.parse(
- fs.readFileSync(path.join(root, '.github/workflows/ci.yml'), 'utf8'),
-);
-const OFFLINE_MEDIA_TEST = '.github/scripts/lib/offline-media.test.mjs';
-const INSTALLS_CHROMIUM = /\bplaywright install(?: --with-deps)? chromium\b/;
-// A suite declared at the start of a line, so a fixture string in a test does not count.
-const GATED_ON_CHROMIUM = /^describe\.runIf\([^)]*\bchromiumInstalled\b/m;
-const BROWSER_SWITCH = /process\.env\.(ASTRYX_[A-Z_]+_BROWSER) === '1'/;
-
-function scriptTests(dir) {
- return fs
- .readdirSync(path.join(root, dir), {withFileTypes: true})
- .flatMap(entry => {
- const rel = path.posix.join(dir, entry.name);
- if (entry.isDirectory()) return scriptTests(rel);
- return entry.name.endsWith('.test.mjs') ? [rel] : [];
- });
-}
-
-/** Every step that runs `file`, with its job's steps and its index there. */
-function stepsRunning(file) {
- const found = [];
- for (const [jobName, job] of Object.entries(ci.jobs)) {
- (job.steps ?? []).forEach((step, index) => {
- if (typeof step.run === 'string' && step.run.includes(file)) {
- found.push({jobName, steps: job.steps, index, step});
- }
- });
- }
- return found;
-}
-
-/** Problems with how ci.yml runs one browser-gated test file. */
-function browserStepProblems(file, source) {
- const browserSwitch = BROWSER_SWITCH.exec(source)?.[1];
- if (browserSwitch == null) {
- return [`${file} has no ASTRYX_*_BROWSER switch to require its browser`];
- }
- const runs = stepsRunning(file).filter(
- ({step}) => step.env?.[browserSwitch] === '1',
- );
- if (runs.length === 0) {
- return [`no ci.yml step runs ${file} with ${browserSwitch}=1`];
- }
- const problems = [];
- for (const {jobName, steps, index, step} of runs) {
- const install = steps
- .slice(0, index)
- .findLast(earlier => INSTALLS_CHROMIUM.test(earlier.run ?? ''));
- if (install == null) {
- problems.push(
- `${jobName}: "${step.name}" runs before any Chromium install`,
- );
- } else if (install.if !== step.if) {
- problems.push(
- `${jobName}: "${step.name}" runs under a different condition than "${install.name}"`,
- );
- }
- }
- return problems;
-}
-
-describe('browser-gated script tests in ci.yml', () => {
- const gated = scriptTests('.github/scripts').filter(file =>
- GATED_ON_CHROMIUM.test(fs.readFileSync(path.join(root, file), 'utf8')),
- );
-
- it('finds the offline-media proof among them', () => {
- expect(gated).toContain(OFFLINE_MEDIA_TEST);
- });
-
- it.each(gated)(
- 'runs %s after a Chromium install, under its condition, with its browser switch',
- file => {
- const source = fs.readFileSync(path.join(root, file), 'utf8');
- expect(browserStepProblems(file, source)).toEqual([]);
- },
- );
-
- it('runs the offline-media cases in the Node test job, with the browser required', () => {
- const [run, ...more] = stepsRunning(OFFLINE_MEDIA_TEST);
- expect(more).toEqual([]);
- expect(run.jobName).toBe('test-node');
- expect(run.step.name).toBe('Offline-media browser cases');
- expect(run.step.env).toEqual({ASTRYX_OFFLINE_MEDIA_BROWSER: '1'});
- expect(run.step.run).toBe(
- `pnpm exec vitest run ${OFFLINE_MEDIA_TEST} --maxWorkers=1`,
- );
- });
-
- it('keeps the full suite, and so the install, for a change to offline-media', () => {
- for (const file of [
- '.github/scripts/lib/offline-media.js',
- OFFLINE_MEDIA_TEST,
- ]) {
- const result = spawnSync(
- 'node',
- [path.join(root, '.github/scripts/change-scope.cjs')],
- {input: `M\t${file}\n`, encoding: 'utf8'},
- );
- const scope = JSON.parse(result.stdout);
- expect({file, ...scope}).toMatchObject({
- file,
- docsiteOnly: false,
- specOnly: false,
- toolingOnly: false,
- fullSuite: true,
- });
- }
- });
-
- describe('the check itself', () => {
- const source =
- "describe.runIf(process.env.ASTRYX_X_BROWSER === '1' || chromiumInstalled)";
- const install = {
- name: 'Install',
- if: 'a',
- run: 'npx playwright install chromium',
- };
- const runStep = (over = {}) => ({
- name: 'Run',
- if: 'a',
- env: {ASTRYX_X_BROWSER: '1'},
- run: 'pnpm exec vitest run x.test.mjs',
- ...over,
- });
- const withJobs = (steps, fn) => {
- const saved = ci.jobs;
- ci.jobs = {job: {steps}};
- try {
- return fn();
- } finally {
- ci.jobs = saved;
- }
- };
-
- it('accepts a run after the install under the same condition', () => {
- expect(
- withJobs([install, runStep()], () =>
- browserStepProblems('x.test.mjs', source),
- ),
- ).toEqual([]);
- });
-
- it.each([
- [
- 'a run before the install',
- [runStep(), install],
- 'runs before any Chromium install',
- ],
- [
- 'a different condition',
- [install, runStep({if: 'b'})],
- 'different condition',
- ],
- [
- 'no browser switch on the step',
- [install, runStep({env: {}})],
- 'no ci.yml step runs x.test.mjs',
- ],
- ['no step at all', [install], 'no ci.yml step runs x.test.mjs'],
- ])('flags %s', (_, steps, problem) => {
- expect(
- withJobs(steps, () => browserStepProblems('x.test.mjs', source)).join(
- '\n',
- ),
- ).toContain(problem);
- });
-
- it('flags a gated file with no browser switch', () => {
- expect(
- browserStepProblems('x.test.mjs', 'describe.runIf(chromiumInstalled)'),
- ).toEqual([
- 'x.test.mjs has no ASTRYX_*_BROWSER switch to require its browser',
- ]);
- });
- });
-});
diff --git a/.github/scripts/vitest.browser-scripts.config.mjs b/.github/scripts/vitest.browser-scripts.config.mjs
new file mode 100644
index 0000000000..1079079711
--- /dev/null
+++ b/.github/scripts/vitest.browser-scripts.config.mjs
@@ -0,0 +1,19 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+/**
+ * @file vitest.browser-scripts.config.mjs
+ * @description Runs one browser-gated script test in a workflow step that has
+ * installed Chromium: no workspace projects and no global setup, so the step
+ * does not rebuild Core for a test that never imports it.
+ * @position The CLI smoke test's CDN page render step.
+ */
+
+import {defineConfig} from 'vitest/config';
+
+export default defineConfig({
+ test: {
+ include: ['.github/scripts/**/*.browser.test.mjs'],
+ // The node project's limit, for slow runners.
+ testTimeout: 30_000,
+ },
+});
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e380b429b2..d6bc66bda6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1150,7 +1150,7 @@ jobs:
# offline-media's real-browser cases. This step runs them once Chromium
# is in place, under the install's own condition; with
# ASTRYX_OFFLINE_MEDIA_BROWSER set, a missing browser fails them.
- # ci-offline-media-browser.test.mjs keeps this step after the install.
+ # ci-browser-gated-tests.test.mjs keeps this step after the install.
- name: Offline-media browser cases
if: matrix.shard == 1 && steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true'
env:
diff --git a/.github/workflows/cli-smoke-test.yml b/.github/workflows/cli-smoke-test.yml
index c10938f4d2..8db7cb4848 100644
--- a/.github/workflows/cli-smoke-test.yml
+++ b/.github/workflows/cli-smoke-test.yml
@@ -27,6 +27,7 @@ on:
- '.github/scripts/api-cli-parity-test.mjs'
- '.github/scripts/cdn-template-smoke-test.mjs'
- '.github/scripts/lib/cdn-*.mjs'
+ - '.github/scripts/vitest.browser-scripts.config.mjs'
- '.github/workflows/cli-smoke-test.yml'
merge_group:
@@ -107,6 +108,18 @@ jobs:
if: steps.scope.outputs.spec_only != 'true'
run: npx playwright install chromium
+ # cdn-page-render's real-browser half: a page whose module request hangs
+ # never goes idle, and renderPage reports that timeout. With
+ # ASTRYX_CDN_SMOKE_BROWSER set, a missing browser fails the cases.
+ # ci-browser-gated-tests.test.mjs keeps this step after the install.
+ - name: CDN page render in a real browser
+ if: steps.scope.outputs.spec_only != 'true'
+ env:
+ ASTRYX_CDN_SMOKE_BROWSER: '1'
+ # Its own config: the workspace's global setup builds Core, which this
+ # test never imports.
+ run: pnpm exec vitest run --config .github/scripts/vitest.browser-scripts.config.mjs .github/scripts/lib/cdn-page-render.browser.test.mjs --maxWorkers=1
+
- name: CDN starter page smoke test
if: steps.scope.outputs.spec_only != 'true'
run: node .github/scripts/cdn-template-smoke-test.mjs