An open source design system that's fully customizable and agent ready
TypeScript
ci: nightly real-Chromium perf scenes on main (#9163)
Cindy Zhangf1b7e0969acd
diff --git a/.github/scripts/ci-pr-gate.test.mjs b/.github/scripts/ci-pr-gate.test.mjsindex f22da8cfff..794c5c70d4 100644--- a/.github/scripts/ci-pr-gate.test.mjs+++ b/.github/scripts/ci-pr-gate.test.mjs@@ -145,6 +145,7 @@ describe('pull-request CI gate (spec:AST-030 FR17)', () => {'crowdin-download.yml','internal-registry.yml','lint.yml',+ 'perf-nightly.yml',]);});diff --git a/.github/scripts/perf-nightly-workflow.test.mjs b/.github/scripts/perf-nightly-workflow.test.mjsnew file mode 100644index 0000000000..cf5a2c5a36--- /dev/null+++ b/.github/scripts/perf-nightly-workflow.test.mjs@@ -0,0 +1,498 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Contract: the nightly perf workflow (.github/workflows/perf-nightly.yml)+ * runs nightly and on dispatch, runs on a pull request only behind the CI+ * gate and a `ci:run-perf` label applied by someone with write access, keeps its+ * jobs on standard runners within the 15-minute budget with minimal token+ * permissions, and opens or updates one regression issue+ * (.github/scripts/perf-regression-issue.mjs). The gate's script is run as+ * actions/github-script runs it, against a stubbed GitHub API.+ */++import fs from 'node:fs';+import {createRequire} from 'node:module';+import path from 'node:path';++import {describe, expect, it} from 'vitest';+import yaml from 'yaml';++import {PR_CI_GATE} from './ci-pr-gate.mjs';+import {+ PERF_ISSUE_TITLE,+ upsertRegressionIssue,+} from './perf-regression-issue.mjs';++const root = path.resolve(import.meta.dirname, '../..');+const workflow = yaml.parse(+ fs.readFileSync(+ path.join(root, '.github/workflows/perf-nightly.yml'),+ 'utf8',+ ),+);+const on = workflow.on ?? workflow[true];+const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor;++describe('triggers', () => {+ it('runs once a day on a fixed schedule, and on manual dispatch', () => {+ expect(on.schedule).toHaveLength(1);+ const fields = on.schedule[0].cron.split(' ');+ expect(fields).toHaveLength(5);+ // A fixed minute and hour, every day of every month.+ expect(fields[0]).toMatch(/^\d+$/);+ expect(fields[1]).toMatch(/^\d+$/);+ expect(fields.slice(2)).toEqual(['*', '*', '*']);+ expect(on).toHaveProperty('workflow_dispatch');+ });++ it('starts on pull request updates only, never on a label (spec:AST-030 FR17)', () => {+ expect(on.pull_request.types).toEqual([+ 'opened',+ 'synchronize',+ 'reopened',+ 'ready_for_review',+ ]);+ });++ it('lists the paths the scenes measure (spec:AST-030 FR14)', () => {+ expect(on.pull_request.paths).toEqual(+ expect.arrayContaining([+ 'packages/core/**',+ 'apps/sandbox/src/app/(raw)/pages/perf-scenes/**',+ 'apps/sandbox/perf/**',+ 'internal/a11y-spec/src/storybook.ts',+ 'playwright.perf.config.ts',+ '.github/workflows/perf-nightly.yml',+ ]),+ );+ for (const glob of on.pull_request.paths) {+ expect(fs.globSync(glob, {cwd: root}).length, glob).toBeGreaterThan(0);+ }+ });+});++describe('the label gate', () => {+ const gate = workflow.jobs.gate;++ /** Evaluates the gate job's condition as Actions would, for one event. */+ function starts({+ event = 'pull_request',+ draft = false,+ fork = false,+ labels = [],+ } = {}) {+ const github = {+ event_name: event,+ repository: 'meta/astryx',+ event: {+ pull_request:+ event === 'pull_request'+ ? {+ draft,+ head: {+ repo: {full_name: fork ? 'someone/astryx' : 'meta/astryx'},+ },+ labels: labels.map(name => ({name})),+ }+ : null,+ },+ };+ const js = gate.if+ .replace(/^\$\{\{\s*|\s*\}\}$/g, '')+ .replace(/\.labels\.\*\.name/g, '.labels.map(l => l.name)');+ return Function(+ 'github',+ 'contains',+ `return Boolean(${js});`,+ )(github, (list, item) => list.includes(item));+ }++ it('begins with the pull request CI gate', () => {+ expect(gate.if.startsWith(`\${{ ${PR_CI_GATE} && (`)).toBe(true);+ });++ it('starts for a labelled pull request, nightly and on dispatch', () => {+ expect(starts({labels: ['ci:run-perf']})).toBe(true);+ expect(starts({fork: true, labels: ['ci:run-perf', 'ci:run']})).toBe(true);+ expect(starts({event: 'schedule'})).toBe(true);+ expect(starts({event: 'workflow_dispatch'})).toBe(true);+ });++ it('skips a pull request without the label, a draft, and an unvouched fork', () => {+ expect(starts()).toBe(false);+ expect(starts({labels: ['ci:heavy-lanes']})).toBe(false);+ expect(starts({draft: true, labels: ['ci:run-perf']})).toBe(false);+ expect(starts({fork: true, labels: ['ci:run-perf']})).toBe(false);+ });++ it('puts every other job behind the gate', () => {+ expect(workflow.jobs['build-scenes'].needs).toEqual(['gate']);+ expect(workflow.jobs['build-scenes'].if).toBe(+ "needs.gate.outputs.run == 'true'",+ );+ expect(workflow.jobs.measure.needs).toEqual(['build-scenes']);+ expect(workflow.jobs.report.needs).toEqual(['measure']);+ for (const id of ['measure', 'report']) {+ expect(workflow.jobs[id].if, id).toBeUndefined();+ }+ });+});++describe("the labeler's access check", () => {+ const script = workflow.jobs.gate.steps.find(step =>+ step.uses?.startsWith('actions/github-script@'),+ ).with.script;+ const realRequire = createRequire(import.meta.url);++ it("reads CI's own label rule from a sparse checkout", () => {+ const checkout = workflow.jobs.gate.steps.find(step =>+ step.uses?.startsWith('actions/checkout@'),+ );+ expect(checkout.with['sparse-checkout'].trim().split('\n')).toEqual([+ '.github/scripts/run-labels.cjs',+ '.github/scripts/review-signal-decision.cjs',+ ]);+ expect(script).toContain("label: 'ci:run-perf'");+ });++ /** Runs the gate script as github-script would; returns `run` and notices. */+ async function decide({+ eventName = 'pull_request',+ repo = 'astryx',+ labels = ['ci:run-perf'],+ events = [{actor: {login: 'maintainer'}, label: {name: 'ci:run-perf'}}],+ permissions = {maintainer: 'write'},+ failLookup = false,+ } = {}) {+ const outputs = {};+ const notices = [];+ const core = {+ setOutput: (name, value) => {+ outputs[name] = value;+ },+ notice: message => notices.push(message),+ warning: message => notices.push(message),+ };+ const github = {+ graphql: async (_query, variables) => {+ if (failLookup) throw new Error('rate limited');+ expect(variables).toEqual({owner: 'meta', repo, number: 7});+ return {+ repository: {pullRequest: {timelineItems: {nodes: events}}},+ };+ },+ rest: {+ repos: {+ getCollaboratorPermissionLevel: async ({username}) => ({+ data: {permission: permissions[username] ?? 'read'},+ }),+ },+ },+ };+ const context = {+ eventName,+ repo: {owner: 'meta', repo},+ payload: {+ pull_request:+ eventName === 'pull_request'+ ? {number: 7, labels: labels.map(name => ({name}))}+ : undefined,+ },+ };+ // The checkout puts the scripts under .run-labels; the tree has them at root.+ const require = file =>+ realRequire(file.replace(`${root}/.run-labels/`, `${root}/`));+ const saved = process.env.GITHUB_WORKSPACE;+ process.env.GITHUB_WORKSPACE = root;+ try {+ await new AsyncFunction('github', 'context', 'core', 'require', script)(+ github,+ context,+ core,+ require,+ );+ } finally {+ process.env.GITHUB_WORKSPACE = saved;+ }+ return {run: outputs.run, outputs, notices};+ }++ it('runs when the label was applied with write or admin access', async () => {+ for (const permission of ['write', 'admin']) {+ expect(+ (await decide({permissions: {maintainer: permission}})).run,+ permission,+ ).toBe('true');+ }+ });++ it('does not run when the labeler has read or no access, and says why', async () => {+ for (const permission of ['read', 'none']) {+ const {run, notices} = await decide({+ permissions: {maintainer: permission},+ });+ expect(run, permission).toBe('false');+ expect(notices.join('\n')).toContain('maintainer');+ }+ });++ it('judges the newest ci:run-perf label event, not an earlier one', async () => {+ const {run} = await decide({+ events: [+ {actor: {login: 'maintainer'}, label: {name: 'ci:run-perf'}},+ {actor: {login: 'maintainer'}, label: {name: 'bug'}},+ {actor: {login: 'reader'}, label: {name: 'ci:run-perf'}},+ ],+ permissions: {maintainer: 'write', reader: 'read'},+ });+ expect(run).toBe('false');+ });++ it('fails closed when the lookup fails or finds no label event', async () => {+ expect((await decide({failLookup: true})).run).toBe('false');+ expect((await decide({events: []})).run).toBe('false');+ });++ it('does not run a pull request without the label, nor for a CI suite label', async () => {+ expect((await decide({labels: ['ci:run']})).run).toBe('false');+ expect((await decide({labels: ['ci:run-visual']})).run).toBe('false');+ });++ it('sets only its own output', async () => {+ expect(Object.keys((await decide()).outputs)).toEqual(['run']);+ });++ it('runs nightly only on meta/astryx, and on every dispatch', async () => {+ expect((await decide({eventName: 'schedule'})).run).toBe('true');+ expect((await decide({eventName: 'schedule', repo: 'fork'})).run).toBe(+ 'false',+ );+ expect((await decide({eventName: 'workflow_dispatch'})).run).toBe('true');+ });+});++describe('jobs', () => {+ const jobs = Object.entries(workflow.jobs);++ // A job named like a required context (`build`, `test`, `lint`, …) would+ // report a second, often skipped, check under that name (spec:AST-030 FR7).+ it('share no name with a required context', () => {+ for (const [id, job] of jobs) {+ for (const name of [id, job.name].filter(Boolean)) {+ expect(+ [+ 'build',+ 'test',+ 'docsite-test',+ 'lint',+ 'spec-owner-approval',+ 'review-required',+ ],+ `${id}`,+ ).not.toContain(name);+ }+ }+ });++ it('use standard runners within the 15-minute budget', () => {+ for (const [id, job] of jobs) {+ expect(['ubuntu-latest', 'ubuntu-slim'], id).toContain(job['runs-on']);+ expect(job['timeout-minutes'], id).toBeLessThanOrEqual(15);+ }+ });++ it('grant the token only what each job uses', () => {+ expect(workflow.permissions).toEqual({});+ expect(workflow.jobs.gate.permissions).toEqual({+ contents: 'read',+ 'pull-requests': 'read',+ });+ expect(workflow.jobs['build-scenes'].permissions).toEqual({+ contents: 'read',+ });+ expect(workflow.jobs.measure.permissions).toEqual({contents: 'read'});+ expect(workflow.jobs.report.permissions).toEqual({+ actions: 'read',+ contents: 'read',+ issues: 'write',+ });+ });++ it('never persists checkout credentials', () => {+ for (const [id, job] of jobs) {+ for (const step of job.steps) {+ if (step.uses?.startsWith('actions/checkout@')) {+ expect(step.with?.['persist-credentials'], id).toBe(false);+ }+ }+ }+ });++ it('uploads the results and the history of every run', () => {+ const uploads = jobs.flatMap(([, job]) =>+ job.steps+ .filter(step => step.uses?.startsWith('actions/upload-artifact@'))+ .map(step => step.with.name),+ );+ expect(uploads).toEqual(['perf-sandbox', 'perf-results', 'perf-history']);+ });++ it('keeps recorded runs on main in one uncancelled group', () => {+ expect(workflow.concurrency.group).toContain("'main'");+ expect(workflow.concurrency['cancel-in-progress']).toBe(+ "${{ github.event_name == 'pull_request' }}",+ );+ });+});++describe('the regression issue', () => {+ const report = {+ sha: 'c'.repeat(40),+ runUrl: 'https://github.com/meta/astryx/actions/runs/1',+ recorded: true,+ range: {from: 'a'.repeat(40), to: 'c'.repeat(40)},+ regressions: [+ {+ scene: 'table',+ metric: 'mountLongTaskMs',+ value: 200,+ median: 100,+ change: 1,+ },+ ],+ };++ function stub(open = []) {+ const calls = [];+ const github = {+ paginate: async (method, params) => method(params),+ rest: {+ issues: {+ listForRepo: async params => {+ calls.push(['list', params]);+ return open;+ },+ create: async params => calls.push(['create', params]),+ createComment: async params => calls.push(['comment', params]),+ },+ repos: {+ compareCommits: async () => ({+ data: {+ total_commits: 2,+ commits: [+ {+ sha: 'a1'.repeat(20),+ commit: {message: 'feat: one (#1)\n\nbody'},+ },+ {sha: 'b2'.repeat(20), commit: {message: 'fix: two (#2)'}},+ ],+ },+ }),+ },+ },+ };+ const context = {repo: {owner: 'meta', repo: 'astryx'}};+ return {github, context, calls};+ }++ it('opens one issue with the regressions and the commit range', async () => {+ const {github, context, calls} = stub();+ expect(await upsertRegressionIssue({github, context, report})).toBe(+ 'created',+ );+ const [, params] = calls.find(([kind]) => kind === 'create');+ expect(params.title).toBe(PERF_ISSUE_TITLE);+ expect(params.body).toContain(+ '| table | mountLongTaskMs | 200 | 100 | +100% |',+ );+ expect(params.body).toContain(+ `compare/${'a'.repeat(40)}...${'c'.repeat(40)}`,+ );+ expect(params.body).toContain('feat: one (#1)');+ expect(params.body).not.toContain('body');+ });++ it('comments on the open issue instead of opening another', async () => {+ const {github, context, calls} = stub([+ {number: 5, title: 'Something else'},+ {number: 6, title: PERF_ISSUE_TITLE, pull_request: {}},+ {number: 7, title: PERF_ISSUE_TITLE},+ ]);+ expect(await upsertRegressionIssue({github, context, report})).toBe(+ 'commented',+ );+ expect(calls.find(([kind]) => kind === 'create')).toBeUndefined();+ expect(calls.find(([kind]) => kind === 'comment')[1].issue_number).toBe(7);+ });++ it('does nothing for a clean or unrecorded run', async () => {+ for (const quiet of [+ {...report, regressions: []},+ {...report, recorded: false},+ ]) {+ const {github, context, calls} = stub();+ expect(+ await upsertRegressionIssue({github, context, report: quiet}),+ ).toBe('none');+ expect(calls).toEqual([]);+ }+ });++ it('states a regression from a zero median as its absolute change', async () => {+ const {github, context, calls} = stub();+ await upsertRegressionIssue({+ github,+ context,+ report: {+ ...report,+ regressions: [+ {+ scene: 'tree-list',+ metric: 'listenerDelta',+ value: 12,+ median: 0,+ change: null,+ },+ ],+ },+ });+ expect(calls.find(([kind]) => kind === 'create')[1].body).toContain(+ '| tree-list | listenerDelta | 12 | 0 | +12 |',+ );+ });++ it('says no commit landed when the last good run is on the same commit', async () => {+ const {github, context, calls} = stub();+ await upsertRegressionIssue({+ github,+ context,+ report: {+ ...report,+ range: null,+ lastGood: {+ sha: 'c'.repeat(40),+ runId: 41,+ date: '2026-11-01T09:30:00Z',+ },+ },+ });+ const {body} = calls.find(([kind]) => kind === 'create')[1];+ expect(body).toContain(+ 'No commit landed since the last good run on ccccccc',+ );+ expect(body).toContain('/actions/runs/41');+ expect(body).not.toContain('No earlier good nightly run');+ });++ it('says so when no good run is recorded yet', async () => {+ const {github, context, calls} = stub();+ await upsertRegressionIssue({+ github,+ context,+ report: {...report, range: null},+ });+ expect(calls.find(([kind]) => kind === 'create')[1].body).toContain(+ 'no commit range yet',+ );+ });+});diff --git a/.github/scripts/perf-regression-issue.mjs b/.github/scripts/perf-regression-issue.mjsnew file mode 100644index 0000000000..609fb464fc--- /dev/null+++ b/.github/scripts/perf-regression-issue.mjs@@ -0,0 +1,106 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file Opens, or adds to, the one open nightly perf regression issue.+ * @input `report` (report.json from perf-trend.mjs), and actions/github-script's+ * `github` and `context`+ * @output An issue titled PERF_ISSUE_TITLE when none is open, else a comment+ * on the open one; nothing when the report has no regression or was not+ * recorded (a pull request or a branch run)+ * @position The report job of .github/workflows/perf-nightly.yml, with+ * `issues: write`. The commit range is the compare view since the last good+ * nightly; its commits are listed (up to MAX_COMMITS) instead of bisecting,+ * because one bisect step costs a full build and measurement.+ */++export const PERF_ISSUE_TITLE = 'Nightly perf regression';+export const MAX_COMMITS = 30;+const BOT = 'github-actions[bot]';++function formatValue(value) {+ return Number.isInteger(value) ? String(value) : value.toFixed(1);+}++/** The Markdown that describes one report's regressions. */+export async function describeReport({github, context, report}) {+ const {owner, repo} = context.repo;+ const lines = [+ `Nightly perf run on ${report.sha.slice(0, 7)} found ${report.regressions.length} metric(s) more than 10% worse than their 7-day median, as on the recorded run before it: [run](${report.runUrl}).`,+ '',+ '| Scene | Metric | This run | 7-day median | Change |',+ '| --- | --- | ---: | ---: | ---: |',+ ...report.regressions.map(+ r =>+ `| ${r.scene} | ${r.metric} | ${formatValue(r.value)} | ${formatValue(r.median)} | ${r.change == null ? `+${formatValue(r.value - r.median)}` : `+${Math.round(r.change * 100)}%`} |`,+ ),+ '',+ ];+ if (report.range == null) {+ const good = report.lastGood;+ lines.push(+ good == null+ ? 'No earlier good nightly run is recorded, so there is no commit range yet.'+ : `No commit landed since the last good run on ${good.sha.slice(0, 7)} ([run](https://github.com/${owner}/${repo}/actions/runs/${good.runId})): the code is the same, so the change comes from the runner.`,+ );+ return lines.join('\n');+ }+ const {from, to} = report.range;+ lines.push(+ `Commits since the last good run: [${from.slice(0, 7)}…${to.slice(0, 7)}](https://github.com/${owner}/${repo}/compare/${from}...${to})`,+ );+ try {+ const {data} = await github.rest.repos.compareCommits({+ owner,+ repo,+ base: from,+ head: to,+ per_page: MAX_COMMITS,+ });+ // Oldest first, as the compare view lists them.+ const commits = data.commits.slice(0, MAX_COMMITS);+ lines.push(+ '',+ ...commits.map(+ commit =>+ `- ${commit.sha.slice(0, 7)} ${commit.commit.message.split('\n')[0]}`,+ ),+ );+ if (data.total_commits > commits.length) {+ lines.push(`- … and ${data.total_commits - commits.length} more`);+ }+ } catch (error) {+ lines.push('', `(The commit list could not be read: ${error.message})`);+ }+ return lines.join('\n');+}++/**+ * Creates the regression issue or comments on the open one. Returns what it+ * did: 'none', 'created' or 'commented'.+ */+export async function upsertRegressionIssue({github, context, report}) {+ if (!report.recorded || report.regressions.length === 0) return 'none';+ const {owner, repo} = context.repo;+ const body = await describeReport({github, context, report});+ const open = await github.paginate(github.rest.issues.listForRepo, {+ owner,+ repo,+ state: 'open',+ creator: BOT,+ per_page: 100,+ });+ const existing = open.find(+ issue => issue.title === PERF_ISSUE_TITLE && issue.pull_request == null,+ );+ if (existing) {+ await github.rest.issues.createComment({+ owner,+ repo,+ issue_number: existing.number,+ body,+ });+ return 'commented';+ }+ await github.rest.issues.create({owner, repo, title: PERF_ISSUE_TITLE, body});+ return 'created';+}diff --git a/.github/scripts/perf-trend.mjs b/.github/scripts/perf-trend.mjsnew file mode 100644index 0000000000..e0bf3cbedd--- /dev/null+++ b/.github/scripts/perf-trend.mjs@@ -0,0 +1,377 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file The nightly perf trend: compare a run's scene medians with the last+ * seven days of main and keep the history.+ * @input `--results <dir>` (one `<scene>.json` per scene from+ * apps/sandbox/perf/scenes.perf.spec.ts), `--history <file>` (the previous+ * run's perf-history.json; may be missing), `--out <dir>`, and the run's+ * `--sha`, `--run-id`, `--run-url`, `--event`, `--ref`, `--repository`,+ * `--date` (ISO).+ * @output In `--out`: `report.json` (regressions, the commit range since the+ * last good run, whether the run was recorded), `summary.md` (for the job+ * summary), and `perf-history.json` (the history with this run appended,+ * when it is a nightly or dispatched run on main; otherwise unchanged).+ * @position The report job of .github/workflows/perf-nightly.yml. The+ * history travels as a workflow artifact from one main run to the next+ * (artifacts need only `actions: read`; the Actions cache is not used for+ * dependencies or data in this repository). Pull request runs compare+ * against the history and record nothing.+ *+ * A metric regresses when it is more than 10% worse than its median over the+ * runs recorded in the seven days before this one, and worse by at least the+ * metric's floor, so noise on a near-zero value (a 1 ms long task becoming+ * 2 ms) is not a regression. A metric with fewer than three runs in the window+ * has no baseline yet. Every metric is better when lower.+ *+ * A standard runner's speed swings by about a fifth from one machine to the+ * next, more than the threshold, so one run over it is only pending: the+ * regression is confirmed, and reported, when the metric was also over it on+ * the recorded run before (the previous night, or a re-run dispatched right+ * after). A run with any metric over the threshold is not a good run.+ *+ * SYNC: METRICS names the fields of SceneSample in apps/sandbox/perf/measure.ts.+ */++import fs from 'node:fs';+import path from 'node:path';+import {pathToFileURL} from 'node:url';++export const THRESHOLD = 0.1;+export const WINDOW_DAYS = 7;+export const MIN_BASELINE_RUNS = 3;+export const HISTORY_LIMIT = 120;++/** metric → the smallest absolute worsening that can count, and its unit. */+export const METRICS = {+ mountLongTaskMs: {floor: 20, unit: 'ms', label: 'Mount long tasks'},+ mountLayouts: {floor: 2, unit: '', label: 'Mount layouts'},+ interactionLayouts: {floor: 2, unit: '', label: 'Interaction layouts'},+ interactionMs: {floor: 16, unit: 'ms', label: 'Interaction latency'},+ heapDeltaKb: {floor: 256, unit: 'KB', label: 'Heap after unmount'},+ listenerDelta: {floor: 5, unit: '', label: 'Listeners after unmount'},+};++const DAY_MS = 24 * 60 * 60 * 1000;++export function median(values) {+ const sorted = [...values].sort((a, b) => a - b);+ const middle = Math.floor(sorted.length / 2);+ return sorted.length % 2 === 1+ ? sorted[middle]+ : (sorted[middle - 1] + sorted[middle]) / 2;+}++/** Whether a run is a nightly or dispatched run on this repository's main. */+export function isRecordable({event, ref, repository}) {+ return (+ (event === 'schedule' || event === 'workflow_dispatch') &&+ ref === 'refs/heads/main' &&+ repository === 'meta/astryx'+ );+}++/** scene → metric → {median, runs} over recorded runs in the window before `at`. */+export function baselineMedians(history, at) {+ const end = Date.parse(at);+ const start = end - WINDOW_DAYS * DAY_MS;+ const values = {};+ for (const run of history.runs) {+ const time = Date.parse(run.date);+ if (!(time >= start && time < end)) continue;+ for (const [scene, metrics] of Object.entries(run.scenes)) {+ for (const [metric, value] of Object.entries(metrics)) {+ if (typeof value !== 'number') continue;+ ((values[scene] ??= {})[metric] ??= []).push(value);+ }+ }+ }+ const baseline = {};+ for (const [scene, metrics] of Object.entries(values)) {+ for (const [metric, list] of Object.entries(metrics)) {+ (baseline[scene] ??= {})[metric] = {+ median: median(list),+ runs: list.length,+ };+ }+ }+ return baseline;+}++/** Whether `value` regresses against `base` (a baseline median) for `metric`. */+export function isRegression(metric, value, base) {+ const floor = METRICS[metric]?.floor ?? 0;+ return value > base * (1 + THRESHOLD) && value - base >= floor;+}++/** Every scene metric that regressed against its baseline. */+export function findRegressions(scenes, baseline) {+ const regressions = [];+ for (const [scene, metrics] of Object.entries(scenes)) {+ for (const metric of Object.keys(METRICS)) {+ const value = metrics[metric];+ const base = baseline[scene]?.[metric];+ if (typeof value !== 'number' || base == null) continue;+ if (base.runs < MIN_BASELINE_RUNS) continue;+ if (isRegression(metric, value, base.median)) {+ regressions.push({+ scene,+ metric,+ value,+ median: base.median,+ change: base.median === 0 ? null : value / base.median - 1,+ });+ }+ }+ }+ return regressions;+}++/** `scene/metric`, the key a run records for each metric over the threshold. */+export function metricKey({scene, metric}) {+ return `${scene}/${metric}`;+}++/**+ * Splits the metrics over the threshold into those also over it on the+ * previous recorded run (confirmed) and those over it for the first time+ * (pending).+ */+export function confirmRegressions(overThreshold, history) {+ const previous = new Set(history.runs.at(-1)?.overThreshold ?? []);+ const confirmed = [];+ const pending = [];+ for (const regression of overThreshold) {+ (previous.has(metricKey(regression)) ? confirmed : pending).push(+ regression,+ );+ }+ return {confirmed, pending};+}++/** The newest recorded run with no metric over the threshold, or null. */+export function lastGoodRun(history) {+ for (let i = history.runs.length - 1; i >= 0; i--) {+ if (history.runs[i].good) return history.runs[i];+ }+ return null;+}++/** The history with `run` appended, oldest runs dropped past HISTORY_LIMIT. */+export function appendRun(history, run) {+ return {version: 1, runs: [...history.runs, run].slice(-HISTORY_LIMIT)};+}++export function readHistory(file) {+ if (file == null || !fs.existsSync(file)) return {version: 1, runs: []};+ const history = JSON.parse(fs.readFileSync(file, 'utf8'));+ if (history?.version !== 1 || !Array.isArray(history.runs)) {+ throw new Error(`${file} is not a version 1 perf history`);+ }+ return history;+}++/** scene → metric → median, from the driver's result files. */+export function readResults(dir) {+ const scenes = {};+ for (const name of fs.readdirSync(dir).sort()) {+ if (!name.endsWith('.json')) continue;+ const result = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));+ if (typeof result.scene !== 'string' || result.median == null) {+ throw new Error(`${name} is not a perf scene result`);+ }+ scenes[result.scene] = result.median;+ }+ if (Object.keys(scenes).length === 0) {+ throw new Error(`no perf scene results in ${dir}`);+ }+ return scenes;+}++function format(metric, value) {+ const unit = METRICS[metric]?.unit ?? '';+ const number = Number.isInteger(value) ? String(value) : value.toFixed(1);+ return unit ? `${number} ${unit}` : number;+}++/** The job summary: every scene's medians beside its baseline. */+export function renderSummary({+ scenes,+ baseline,+ regressions,+ pending = [],+ range,+ recorded = true,+}) {+ // A run that is not recorded (a pull request's) confirms nothing: every+ // metric over the threshold is simply over it.+ const confirmed = new Set(+ (recorded ? regressions : [...regressions, ...pending]).map(metricKey),+ );+ const first = new Set(recorded ? pending.map(metricKey) : []);+ const over = regressions.length + pending.length;+ const headline = recorded+ ? [+ regressions.length === 0+ ? 'No metric is more than 10% worse than its 7-day median on two recorded runs in a row.'+ : `**${regressions.length} metric(s) regressed** against the 7-day median on two recorded runs in a row.`,+ ...(pending.length === 0+ ? []+ : [+ '',+ `${pending.length} metric(s) are over the threshold for the first time (⏳); the next recorded run confirms or clears them.`,+ ]),+ ]+ : [+ over === 0+ ? "Compared with `main`'s 7-day median: no metric is more than 10% worse. This run is not recorded."+ : `Compared with \`main\`'s 7-day median: **${over} metric(s) are more than 10% worse** (⚠️). This run is not recorded.`,+ ];+ const lines = [+ '## Nightly perf scenes',+ '',+ ...headline,+ '',+ `| Scene | ${Object.values(METRICS)+ .map(m => m.label)+ .join(' | ')} |`,+ `| --- |${' ---: |'.repeat(Object.keys(METRICS).length)}`,+ ];+ for (const [scene, metrics] of Object.entries(scenes)) {+ const cells = Object.keys(METRICS).map(metric => {+ const value = metrics[metric];+ if (typeof value !== 'number') return '—';+ const base = baseline[scene]?.[metric];+ const key = metricKey({scene, metric});+ const mark = confirmed.has(key) ? ' ⚠️' : first.has(key) ? ' ⏳' : '';+ return base && base.runs >= MIN_BASELINE_RUNS+ ? `${format(metric, value)} (7d ${format(metric, base.median)})${mark}`+ : format(metric, value);+ });+ lines.push(`| ${scene} | ${cells.join(' | ')} |`);+ }+ if (range) {+ lines.push(+ '',+ `Commits since the last good run: ${range.from.slice(0, 7)}…${range.to.slice(0, 7)}`,+ );+ }+ return `${lines.join('\n')}\n`;+}++/** Compares a run with the history; returns the report and the next history. */+export function evaluateRun({scenes, history, run}) {+ const baseline = baselineMedians(history, run.date);+ const overThreshold = findRegressions(scenes, baseline);+ const {confirmed: regressions, pending} = confirmRegressions(+ overThreshold,+ history,+ );+ const good = lastGoodRun(history);+ const range =+ regressions.length > 0 && good != null && good.sha !== run.sha+ ? {from: good.sha, to: run.sha}+ : null;+ const recorded = isRecordable(run);+ const nextHistory = recorded+ ? appendRun(history, {+ sha: run.sha,+ runId: run.runId,+ date: run.date,+ good: overThreshold.length === 0,+ overThreshold: overThreshold.map(metricKey),+ scenes,+ })+ : history;+ return {+ report: {+ sha: run.sha,+ runUrl: run.runUrl,+ recorded,+ lastGood: good+ ? {sha: good.sha, runId: good.runId, date: good.date}+ : null,+ range,+ regressions,+ pending,+ },+ baseline,+ nextHistory,+ };+}++/**+ * Reads a run's results and the previous history, writes report.json,+ * perf-history.json and summary.md to `outDir`, and returns the report and+ * the summary. The workflow calls it from actions/github-script; the CLI below+ * is the same call.+ */+export function runReport({resultsDir, historyFile, outDir, run}) {+ const scenes = readResults(resultsDir);+ const history = readHistory(historyFile);+ const {report, baseline, nextHistory} = evaluateRun({scenes, history, run});+ const summary = renderSummary({+ scenes,+ baseline,+ regressions: report.regressions,+ pending: report.pending,+ range: report.range,+ recorded: report.recorded,+ });+ fs.mkdirSync(outDir, {recursive: true});+ fs.writeFileSync(+ path.join(outDir, 'report.json'),+ `${JSON.stringify(report, null, 2)}\n`,+ );+ fs.writeFileSync(+ path.join(outDir, 'perf-history.json'),+ `${JSON.stringify(nextHistory, null, 2)}\n`,+ );+ fs.writeFileSync(path.join(outDir, 'summary.md'), summary);+ return {report, summary, earlierRuns: history.runs.length};+}++function parseArgs(argv) {+ const args = {};+ for (let i = 0; i < argv.length; i += 2) {+ if (!argv[i].startsWith('--')) throw new Error(`unexpected ${argv[i]}`);+ args[argv[i].slice(2)] = argv[i + 1];+ }+ for (const required of [+ 'results',+ 'out',+ 'sha',+ 'event',+ 'ref',+ 'repository',+ 'date',+ ]) {+ if (!args[required]) throw new Error(`--${required} is required`);+ }+ return args;+}++if (+ process.argv[1] &&+ import.meta.url === pathToFileURL(process.argv[1]).href+) {+ const args = parseArgs(process.argv.slice(2));+ const {report, earlierRuns} = runReport({+ resultsDir: args.results,+ historyFile: args.history,+ outDir: args.out,+ run: {+ sha: args.sha,+ runId: args['run-id'] ?? null,+ runUrl: args['run-url'] ?? null,+ event: args.event,+ ref: args.ref,+ repository: args.repository,+ date: args.date,+ },+ });+ console.log(+ `${report.regressions.length} regression(s), ${report.recorded ? 'recorded' : 'not recorded'}; ${earlierRuns} earlier run(s) in the history.`,+ );+}diff --git a/.github/scripts/perf-trend.test.mjs b/.github/scripts/perf-trend.test.mjsnew file mode 100644index 0000000000..078ca6e531--- /dev/null+++ b/.github/scripts/perf-trend.test.mjs@@ -0,0 +1,379 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file The nightly perf trend's threshold, baseline window and history+ * (.github/scripts/perf-trend.mjs).+ */++import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';++import {describe, expect, it} from 'vitest';++import {+ HISTORY_LIMIT,+ METRICS,+ MIN_BASELINE_RUNS,+ appendRun,+ baselineMedians,+ evaluateRun,+ findRegressions,+ isRecordable,+ isRegression,+ lastGoodRun,+ median,+ readResults,+ renderSummary,+ runReport,+} from './perf-trend.mjs';++const NOW = '2026-11-08T09:30:00.000Z';+const daysBefore = days =>+ new Date(Date.parse(NOW) - days * 24 * 60 * 60 * 1000).toISOString();++const SCENE = {+ mountLongTaskMs: 100,+ mountLayouts: 10,+ interactionLayouts: 10,+ interactionMs: 80,+ heapDeltaKb: 1000,+ listenerDelta: 0,+};++function run(days, scenes = {table: SCENE}, extra = {}) {+ return {+ sha: `sha-${days}`,+ runId: days,+ date: daysBefore(days),+ good: true,+ scenes,+ ...extra,+ };+}++const NIGHTLY = {+ sha: 'sha-now',+ runId: 99,+ runUrl: 'https://example.test/run/99',+ event: 'schedule',+ ref: 'refs/heads/main',+ repository: 'meta/astryx',+ date: NOW,+};++describe('median', () => {+ it('takes the middle value, or the mean of the middle two', () => {+ expect(median([5, 1, 3])).toBe(3);+ expect(median([4, 1, 3, 2])).toBe(2.5);+ });+});++describe('isRegression', () => {+ it('needs more than 10% worse than the median', () => {+ expect(isRegression('mountLongTaskMs', 330, 300)).toBe(false);+ expect(isRegression('mountLongTaskMs', 331, 300)).toBe(true);+ });++ it('needs at least the metric floor of absolute worsening', () => {+ // +50% but only 1 ms: noise on a near-zero value.+ expect(isRegression('mountLongTaskMs', 3, 2)).toBe(false);+ expect(isRegression('mountLongTaskMs', 22, 2)).toBe(true);+ expect(isRegression('mountLayouts', 3, 2)).toBe(false);+ expect(isRegression('mountLayouts', 4, 2)).toBe(true);+ expect(isRegression('heapDeltaKb', 1255, 1000)).toBe(false);+ expect(isRegression('heapDeltaKb', 1256, 1000)).toBe(true);+ expect(isRegression('listenerDelta', 4, 0)).toBe(false);+ expect(isRegression('listenerDelta', 5, 0)).toBe(true);+ });++ it('never flags an improvement', () => {+ for (const metric of Object.keys(METRICS)) {+ expect(isRegression(metric, 0, 1000), metric).toBe(false);+ }+ });+});++describe('baselineMedians', () => {+ it('reads only runs in the 7 days before the run', () => {+ const history = {+ version: 1,+ runs: [+ run(8, {table: {...SCENE, mountLayouts: 1000}}),+ run(6, {table: {...SCENE, mountLayouts: 10}}),+ run(3, {table: {...SCENE, mountLayouts: 12}}),+ run(1, {table: {...SCENE, mountLayouts: 14}}),+ // A run recorded at or after this one's time is not its baseline.+ {...run(0, {table: {...SCENE, mountLayouts: 2000}}), date: NOW},+ ],+ };+ expect(baselineMedians(history, NOW).table.mountLayouts).toEqual({+ median: 12,+ runs: 3,+ });+ });+});++describe('findRegressions', () => {+ const baseline = runs => ({+ table: Object.fromEntries(+ Object.entries(SCENE).map(([metric, value]) => [+ metric,+ {median: value, runs},+ ]),+ ),+ });++ it('names each metric past threshold and floor', () => {+ const regressions = findRegressions(+ {table: {...SCENE, mountLayouts: 20, interactionMs: 85}},+ baseline(MIN_BASELINE_RUNS),+ );+ expect(regressions).toEqual([+ {+ scene: 'table',+ metric: 'mountLayouts',+ value: 20,+ median: 10,+ change: 1,+ },+ ]);+ });++ it(`waits for ${MIN_BASELINE_RUNS} runs in the window`, () => {+ expect(+ findRegressions(+ {table: {...SCENE, mountLayouts: 20}},+ baseline(MIN_BASELINE_RUNS - 1),+ ),+ ).toEqual([]);+ });++ it('skips a scene with no baseline (a new scene)', () => {+ expect(+ findRegressions({markdown: SCENE}, baseline(MIN_BASELINE_RUNS)),+ ).toEqual([]);+ });++ it('reports a regression from a zero median without a ratio', () => {+ const [regression] = findRegressions(+ {table: {...SCENE, listenerDelta: 12}},+ baseline(MIN_BASELINE_RUNS),+ );+ expect(regression).toMatchObject({metric: 'listenerDelta', change: null});+ });+});++describe('isRecordable', () => {+ it('records nightly and dispatched runs on main of this repository only', () => {+ expect(isRecordable(NIGHTLY)).toBe(true);+ expect(isRecordable({...NIGHTLY, event: 'workflow_dispatch'})).toBe(true);+ expect(isRecordable({...NIGHTLY, event: 'pull_request'})).toBe(false);+ expect(isRecordable({...NIGHTLY, event: 'push'})).toBe(false);+ expect(isRecordable({...NIGHTLY, ref: 'refs/heads/feature'})).toBe(false);+ expect(isRecordable({...NIGHTLY, repository: 'someone/astryx'})).toBe(+ false,+ );+ });+});++describe('history', () => {+ it('finds the newest good run', () => {+ const history = {+ version: 1,+ runs: [run(3), run(2), run(1, undefined, {good: false})],+ };+ expect(lastGoodRun(history).sha).toBe('sha-2');+ expect(lastGoodRun({version: 1, runs: []})).toBeNull();+ });++ it(`keeps the newest ${HISTORY_LIMIT} runs`, () => {+ let history = {version: 1, runs: []};+ for (let i = 0; i < HISTORY_LIMIT + 5; i++) {+ history = appendRun(history, {...run(1), sha: `sha-${i}`});+ }+ expect(history.runs).toHaveLength(HISTORY_LIMIT);+ expect(history.runs.at(-1).sha).toBe(`sha-${HISTORY_LIMIT + 4}`);+ expect(history.runs[0].sha).toBe('sha-5');+ });+});++describe('evaluateRun', () => {+ // The newest recorded run was already over the threshold on Table's mount.+ const history = {+ version: 1,+ runs: [+ run(4),+ run(3),+ run(2),+ run(1, undefined, {+ good: false,+ overThreshold: ['table/mountLongTaskMs'],+ }),+ ],+ };+ const firstTime = {+ version: 1,+ runs: [run(4), run(3), run(2), run(1)],+ };++ it('confirms a metric over the threshold on two recorded runs in a row, with the commits since the last good run', () => {+ const {report, nextHistory} = evaluateRun({+ scenes: {table: {...SCENE, mountLongTaskMs: 200}},+ history,+ run: NIGHTLY,+ });+ expect(report.regressions.map(r => r.metric)).toEqual(['mountLongTaskMs']);+ expect(report.pending).toEqual([]);+ expect(report.range).toEqual({from: 'sha-2', to: 'sha-now'});+ expect(report.recorded).toBe(true);+ expect(nextHistory.runs.at(-1)).toMatchObject({+ sha: 'sha-now',+ good: false,+ overThreshold: ['table/mountLongTaskMs'],+ });+ });++ it('holds a metric over the threshold for the first time as pending, and records it', () => {+ const {report, nextHistory} = evaluateRun({+ scenes: {table: {...SCENE, mountLongTaskMs: 200}},+ history: firstTime,+ run: NIGHTLY,+ });+ expect(report.regressions).toEqual([]);+ expect(report.pending.map(r => r.metric)).toEqual(['mountLongTaskMs']);+ expect(report.range).toBeNull();+ expect(nextHistory.runs.at(-1)).toMatchObject({+ good: false,+ overThreshold: ['table/mountLongTaskMs'],+ });+ });++ it('does not confirm a different metric than the previous run had over', () => {+ const {report} = evaluateRun({+ scenes: {table: {...SCENE, mountLayouts: 20}},+ history,+ run: NIGHTLY,+ });+ expect(report.regressions).toEqual([]);+ expect(report.pending.map(r => r.metric)).toEqual(['mountLayouts']);+ });++ it('reads a run recorded before confirmation as having nothing over', () => {+ const {report} = evaluateRun({+ scenes: {table: {...SCENE, mountLongTaskMs: 200}},+ history: firstTime,+ run: NIGHTLY,+ });+ expect(report.regressions).toEqual([]);+ });++ it('records a clean run as good, with no range', () => {+ const {report, nextHistory} = evaluateRun({+ scenes: {table: SCENE},+ history,+ run: NIGHTLY,+ });+ expect(report.regressions).toEqual([]);+ expect(report.pending).toEqual([]);+ expect(report.range).toBeNull();+ expect(nextHistory.runs.at(-1)).toMatchObject({+ good: true,+ overThreshold: [],+ });+ });++ it('compares a pull request run without recording it', () => {+ const {report, nextHistory} = evaluateRun({+ scenes: {table: {...SCENE, mountLongTaskMs: 200}},+ history,+ run: {...NIGHTLY, event: 'pull_request', ref: 'refs/pull/1/merge'},+ });+ expect(report.regressions).toHaveLength(1);+ expect(report.recorded).toBe(false);+ expect(nextHistory).toBe(history);+ });+});++describe('runReport', () => {+ it('reads the driver results, starts a history when none exists, and writes the report', () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'perf-trend-'));+ const results = path.join(dir, 'results');+ fs.mkdirSync(results);+ fs.writeFileSync(+ path.join(results, 'table.json'),+ JSON.stringify({scene: 'table', samples: [SCENE], median: SCENE}),+ );+ const out = path.join(dir, 'out');+ const {report, summary, earlierRuns} = runReport({+ resultsDir: results,+ historyFile: path.join(dir, 'missing.json'),+ outDir: out,+ run: NIGHTLY,+ });+ expect(earlierRuns).toBe(0);+ expect(report.recorded).toBe(true);+ expect(summary).toContain('| table |');+ const written = JSON.parse(+ fs.readFileSync(path.join(out, 'perf-history.json'), 'utf8'),+ );+ expect(written.runs).toHaveLength(1);+ expect(fs.existsSync(path.join(out, 'report.json'))).toBe(true);+ fs.rmSync(dir, {recursive: true, force: true});+ });++ it('refuses an empty results directory', () => {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'perf-trend-'));+ expect(() => readResults(dir)).toThrow(/no perf scene results/);+ fs.rmSync(dir, {recursive: true, force: true});+ });+});++describe('renderSummary', () => {+ it('marks every metric over the threshold alike on a run that is not recorded', () => {+ const summary = renderSummary({+ scenes: {table: {...SCENE, mountLayouts: 20, interactionMs: 200}},+ baseline: {+ table: {+ mountLayouts: {median: 10, runs: 5},+ interactionMs: {median: 80, runs: 5},+ },+ },+ regressions: [{scene: 'table', metric: 'mountLayouts'}],+ pending: [{scene: 'table', metric: 'interactionMs'}],+ range: null,+ recorded: false,+ });+ expect(summary).toContain('**2 metric(s) are more than 10% worse**');+ expect(summary).toContain('This run is not recorded.');+ expect(summary).toContain('20 (7d 10) ⚠️');+ expect(summary).toContain('200 ms (7d 80 ms) ⚠️');+ expect(summary).not.toContain('⏳');+ expect(summary).not.toContain('next recorded run');+ });++ it('marks a metric over the threshold for the first time as pending', () => {+ const summary = renderSummary({+ scenes: {table: {...SCENE, mountLayouts: 20}},+ baseline: {table: {mountLayouts: {median: 10, runs: 5}}},+ regressions: [],+ pending: [{scene: 'table', metric: 'mountLayouts'}],+ range: null,+ });+ expect(summary).toContain('20 (7d 10) ⏳');+ expect(summary).toContain(+ '1 metric(s) are over the threshold for the first time',+ );+ });++ it('marks a regressed metric and shows its 7-day median', () => {+ const summary = renderSummary({+ scenes: {table: {...SCENE, mountLayouts: 20}},+ baseline: {table: {mountLayouts: {median: 10, runs: 5}}},+ regressions: [{scene: 'table', metric: 'mountLayouts'}],+ range: {from: 'aaaaaaaaaa', to: 'bbbbbbbbbb'},+ });+ expect(summary).toContain('20 (7d 10) ⚠️');+ expect(summary).toContain('aaaaaaa…bbbbbbb');+ });+});diff --git a/.github/scripts/run-labels.cjs b/.github/scripts/run-labels.cjsindex 2a549ce9ac..5bc0d02bde 100644--- a/.github/scripts/run-labels.cjs+++ b/.github/scripts/run-labels.cjs@@ -2,7 +2,7 @@/*** @file Decides which heavy suites a pull request opted into with a- * `ci:run-<suite>` label (spec:AST-030 FR12, DEC-7).+ * `ci:run-<suite>` label (spec:AST-030 FR12, FR18, DEC-7).** A suite runs on a pull request only while the pull request carries its* label, and only when the newest application of that label came from someone@@ -69,17 +69,17 @@ async function newestLabelers(github, {owner, repo, number}) {}/**- * Decides every suite and sets one `true`/`false` output per suite.+ * Decides each suite in `suites` (suite → label) without setting outputs.* @returns {Promise<Record<string, boolean>>}*/-async function decideRunLabels({github, context, core}) {+async function decideSuites({github, context, core, suites}) {const decisions = Object.fromEntries(- Object.keys(SUITE_LABELS).map(suite => [suite, false]),+ Object.keys(suites).map(suite => [suite, false]),);const pr = context.payload.pull_request;if (context.eventName === 'pull_request' && pr != null) {const carried = new Set((pr.labels ?? []).map(label => label.name));- const wanted = Object.entries(SUITE_LABELS).filter(([, label]) =>+ const wanted = Object.entries(suites).filter(([, label]) =>carried.has(label),);if (wanted.length > 0) {@@ -136,10 +136,44 @@ async function decideRunLabels({github, context, core}) {}}}+ return decisions;+}++/**+ * Decides every CI suite and sets one `true`/`false` output per suite.+ * @returns {Promise<Record<string, boolean>>}+ */+async function decideRunLabels({github, context, core}) {+ const decisions = await decideSuites({+ github,+ context,+ core,+ suites: SUITE_LABELS,+ });for (const [suite, runs] of Object.entries(decisions)) {core.setOutput(suite, String(runs));}return decisions;}-module.exports = {SUITE_LABELS, decideRunLabels, notRunSentence};+/**+ * Decides one suite that lives in its own workflow (the nightly perf lane's+ * `ci:run-perf`), under the same rule as the CI suites. Sets no output.+ * @returns {Promise<boolean>}+ */+async function decideRunLabel({github, context, core, suite, label}) {+ const decisions = await decideSuites({+ github,+ context,+ core,+ suites: {[suite]: label},+ });+ return decisions[suite];+}++module.exports = {+ SUITE_LABELS,+ decideRunLabel,+ decideRunLabels,+ notRunSentence,+};diff --git a/.github/scripts/run-labels.test.mjs b/.github/scripts/run-labels.test.mjsindex 4e7a0d180c..16fa92e8f4 100644--- a/.github/scripts/run-labels.test.mjs+++ b/.github/scripts/run-labels.test.mjs@@ -17,6 +17,7 @@ import {ungated} from './ci-pr-gate.mjs';const require = createRequire(import.meta.url);const {SUITE_LABELS,+ decideRunLabel,decideRunLabels,notRunSentence,} = require('./run-labels.cjs');@@ -196,6 +197,80 @@ describe('run-labels decision', () => {});});+// A suite in its own workflow (the nightly perf lane) asks about one label+// under the same rule and sets no output of its own (spec:AST-030 FR18).+describe('decideRunLabel', () => {+ async function decideOne({+ labels,+ permission = 'write',+ eventName = 'pull_request',+ }) {+ const outputs = {};+ const notices = [];+ const core = {+ setOutput: (name, value) => {+ outputs[name] = value;+ },+ notice: message => notices.push(message),+ warning: message => notices.push(message),+ };+ const github = {+ graphql: async () => ({+ repository: {+ pullRequest: {+ timelineItems: {+ nodes: labels.map(name => ({+ actor: {login: 'maintainer'},+ label: {name},+ })),+ },+ },+ },+ }),+ rest: {+ repos: {+ getCollaboratorPermissionLevel: async () => ({data: {permission}}),+ },+ },+ };+ const context = {+ eventName,+ repo: {owner: 'meta', repo: 'astryx'},+ payload: {+ pull_request: {number: 7, labels: labels.map(name => ({name}))},+ },+ };+ const runs = await decideRunLabel({+ github,+ context,+ core,+ suite: 'perf',+ label: 'ci:run-perf',+ });+ return {runs, outputs, notices};+ }++ it('runs for its label from write access, and sets no output', async () => {+ const {runs, outputs} = await decideOne({labels: ['ci:run-perf']});+ expect(runs).toBe(true);+ expect(outputs).toEqual({});+ });++ it('does not run for read access, another suite label, or outside a pull request', async () => {+ expect(+ (await decideOne({labels: ['ci:run-perf'], permission: 'read'})).runs,+ ).toBe(false);+ expect((await decideOne({labels: ['ci:run-visual']})).runs).toBe(false);+ expect(+ (await decideOne({labels: ['ci:run-perf'], eventName: 'push'})).runs,+ ).toBe(false);+ });++ it('leaves the CI suites unchanged', () => {+ expect(Object.values(SUITE_LABELS)).not.toContain('ci:run-perf');+ });+});+// Replay: each label's decision through every heavy owner's condition on a// pull request whose other needs succeeded.describe('each label runs only its own suite', () => {diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.ymlnew file mode 100644index 0000000000..f08061d4ec--- /dev/null+++ b/.github/workflows/perf-nightly.yml@@ -0,0 +1,270 @@+# Copyright (c) Meta Platforms, Inc. and affiliates.++# Nightly perf scenes: real-Chromium mount, interaction and leak numbers for+# five fixed scenes (apps/sandbox/src/app/(raw)/pages/perf-scenes/), measured+# on main every night and compared with main's own 7-day median.+#+# - Runs nightly on main and by manual dispatch (spec:AST-030 FR18). A pull+# request runs it only while it carries the `ci:run-perf` label last applied+# by someone with write, maintain or admin access, decided by CI's own+# run-labels.cjs; like the `ci:run-*` suite labels, it takes effect on the+# next push or a close and reopen, and applying a label starts no run+# (FR12, FR17). The pull request's paths must also touch what the+# scenes measure (spec:AST-030 FR14).+# - build-scenes: the packages and the Sandbox, once (not `build`, which is+# CI's required context). measure: every scene, five+# samples after a warm-up, the median kept (playwright.perf.config.ts).+# report: compares each metric with the 7-day median of main's recorded+# runs; a nightly or dispatched run on main appends itself to the history+# and, when a metric is more than 10% worse (and past the metric's floor)+# on it and on the recorded run before it, opens the "Nightly perf+# regression" issue or comments on the open one,+# with the commits since the last good run (.github/scripts/perf-trend.mjs,+# .github/scripts/perf-regression-issue.mjs).+# - The history is the `perf-history` artifact, carried from each recorded run+# to the next: it needs only `actions: read`, no branch to write and no+# Actions cache (which this repository keeps off). A gap longer than the+# artifact's retention starts a fresh history.+# - Every job keeps the 15-minute budget on standard runners; split a job that+# nears it (shard measure by scene), never raise it.+name: Nightly perf++on:+ schedule:+ - cron: '17 9 * * *'+ workflow_dispatch:+ pull_request:+ types: [opened, synchronize, reopened, ready_for_review]+ branches: ['main']+ # What the scenes measure or run (spec:AST-030 FR14): Core and the build+ # plugin that compiles its styles, the default theme, the scenes and their+ # fixtures, the driver and its config, the trend and label scripts and this+ # workflow.+ paths:+ - 'packages/core/**'+ - 'packages/build/**'+ - 'packages/themes/neutral/**'+ - 'apps/sandbox/src/app/(raw)/pages/perf-scenes/**'+ - 'apps/sandbox/src/app/(fullscreen)/pages/markdown-perf/benchmark.ts'+ - 'apps/sandbox/src/app/(sandbox)/pages/table-lab/tableLabData.ts'+ - 'apps/sandbox/perf/**'+ - 'internal/interaction-harness/src/**'+ - 'internal/a11y-spec/src/storybook.ts'+ - 'playwright.perf.config.ts'+ - '.github/scripts/perf-*.mjs'+ - '.github/scripts/run-labels.cjs'+ - '.github/scripts/review-signal-decision.cjs'+ - '.github/workflows/perf-nightly.yml'+ - 'package.json'+ - 'pnpm-lock.yaml'++permissions: {}++# A pull request's newer push replaces its running measurement. Recorded runs+# on main share one group and never cancel each other, so two runs cannot+# both extend the same history.+concurrency:+ group: "perf-${{ github.event.pull_request.number || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && 'main') || github.run_id }}"+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++jobs:+ gate:+ if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:run-perf')) }}+ runs-on: ubuntu-slim+ timeout-minutes: 5+ permissions:+ contents: read+ pull-requests: read+ outputs:+ run: ${{ steps.decide.outputs.run }}+ steps:+ # The label rule is CI's own (run-labels.cjs, with the review gate's+ # write-access check), from the merge commit's trusted sparse checkout.+ - name: Check out the decision script+ uses: actions/checkout@v7+ with:+ sparse-checkout: |+ .github/scripts/run-labels.cjs+ .github/scripts/review-signal-decision.cjs+ sparse-checkout-cone-mode: false+ path: .run-labels+ persist-credentials: false++ - name: Decide whether to measure+ id: decide+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ const {owner, repo} = context.repo;+ if (context.eventName === 'schedule') {+ // A fork's copy of the schedule measures nothing.+ const ours = `${owner}/${repo}` === 'meta/astryx';+ if (!ours) core.notice('Nightly perf runs only on meta/astryx.');+ core.setOutput('run', String(ours));+ return;+ }+ if (context.eventName === 'workflow_dispatch') {+ core.setOutput('run', 'true');+ return;+ }+ const {decideRunLabel} = require(+ `${process.env.GITHUB_WORKSPACE}/.run-labels/.github/scripts/run-labels.cjs`,+ );+ const runs = await decideRunLabel({+ github, context, core, suite: 'perf', label: 'ci:run-perf',+ });+ core.setOutput('run', String(runs));++ build-scenes:+ needs: [gate]+ if: needs.gate.outputs.run == 'true'+ runs-on: ubuntu-latest+ timeout-minutes: 15+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@v7+ with:+ persist-credentials: false++ - name: Setup Node and pnpm+ uses: ./.github/actions/setup++ - name: Build the packages+ run: pnpm build++ - name: Build the Sandbox+ run: pnpm -F @astryxdesign/sandbox build++ - name: Upload the built Sandbox+ uses: actions/upload-artifact@v7+ with:+ name: perf-sandbox+ path: apps/sandbox/out+ retention-days: 1+ if-no-files-found: error++ measure:+ needs: [build-scenes]+ runs-on: ubuntu-latest+ timeout-minutes: 15+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@v7+ with:+ persist-credentials: false++ - name: Setup Node and pnpm+ uses: ./.github/actions/setup++ - name: Install Chromium+ run: npx playwright install chromium++ - name: Download the built Sandbox+ uses: actions/download-artifact@v8+ with:+ name: perf-sandbox+ path: apps/sandbox/out++ - name: Measure the scenes+ env:+ PERF_RESULTS_DIR: perf-results+ run: pnpm exec playwright test --config playwright.perf.config.ts++ - name: Upload the scene results+ uses: actions/upload-artifact@v7+ with:+ name: perf-results+ path: perf-results+ retention-days: 30+ if-no-files-found: error++ report:+ needs: [measure]+ runs-on: ubuntu-slim+ timeout-minutes: 5+ permissions:+ actions: read+ contents: read+ issues: write+ steps:+ - uses: actions/checkout@v7+ with:+ sparse-checkout: .github/scripts+ persist-credentials: false++ - name: Find the last recorded history+ id: previous+ uses: actions/github-script@v9+ with:+ script: |+ const {owner, repo} = context.repo;+ const {data} = await github.rest.actions.listWorkflowRuns({+ owner, repo, workflow_id: 'perf-nightly.yml', branch: 'main',+ status: 'success', per_page: 20,+ });+ for (const run of data.workflow_runs) {+ if (run.id === context.runId) continue;+ if (!['schedule', 'workflow_dispatch'].includes(run.event)) continue;+ const {data: artifacts} = await github.rest.actions.listWorkflowRunArtifacts({+ owner, repo, run_id: run.id, name: 'perf-history',+ });+ if (artifacts.artifacts.some(artifact => !artifact.expired)) {+ core.setOutput('run-id', String(run.id));+ return;+ }+ }+ core.notice('No earlier perf history; this run starts one.');++ - name: Download the last recorded history+ if: steps.previous.outputs.run-id != ''+ uses: actions/download-artifact@v8+ with:+ name: perf-history+ path: previous-history+ run-id: ${{ steps.previous.outputs.run-id }}+ github-token: ${{ github.token }}++ - name: Download the scene results+ uses: actions/download-artifact@v8+ with:+ name: perf-results+ path: perf-results++ - name: Compare with the 7-day median and report+ uses: actions/github-script@v9+ with:+ script: |+ const workspace = process.env.GITHUB_WORKSPACE;+ const {runReport} = await import(`${workspace}/.github/scripts/perf-trend.mjs`);+ const {upsertRegressionIssue} = await import(+ `${workspace}/.github/scripts/perf-regression-issue.mjs`+ );+ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;+ const {report, summary} = runReport({+ resultsDir: `${workspace}/perf-results`,+ historyFile: `${workspace}/previous-history/perf-history.json`,+ outDir: `${workspace}/perf-report`,+ run: {+ sha: context.sha,+ runId: context.runId,+ runUrl,+ event: context.eventName,+ ref: context.ref,+ repository: `${context.repo.owner}/${context.repo.repo}`,+ date: new Date().toISOString(),+ },+ });+ await core.summary.addRaw(summary).write();+ const outcome = await upsertRegressionIssue({github, context, report});+ core.info(`Regression issue: ${outcome}.`);++ - name: Upload the history+ uses: actions/upload-artifact@v7+ with:+ name: perf-history+ path: perf-report/perf-history.json+ retention-days: 90+ if-no-files-found: errordiff --git a/apps/sandbox/perf/measure.ts b/apps/sandbox/perf/measure.tsindex 96b2b73270..7298d18399 100644--- a/apps/sandbox/perf/measure.ts+++ b/apps/sandbox/perf/measure.ts@@ -10,6 +10,9 @@* (`traceRendering`); heap and listener counts from the DevTools protocol* after a forced garbage collection; interaction latency from the page's* own Event Timing entries, the measurement behind INP.+ *+ * SYNC: Keep the metric names equal to METRICS in+ * .github/scripts/perf-trend.mjs.*/import type {CDPSession, Page} from '@playwright/test';@@ -100,7 +103,10 @@ async function unmount(page: Page): Promise<void> {* one in the same page and returns that sample. The first cycle pays for* loading and compiling the scene's code and for caches that live past an* unmount, so the sample is a warm mount, and its heap and listener deltas- * are what one more cycle leaves behind. The page must be freshly loaded on+ * are what one more cycle leaves behind. The measured cycle runs under+ * `traceRendering` and the warm-up does not, so the heap figure also holds the+ * trace's own retained objects (tens of KB, the same on every run), which+ * shifts the level but not the trend. The page must be freshly loaded on* `?scene=<id>`.*/export async function measureScene(diff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.mdindex f355701770..0ba73a9f30 100644--- a/docs/specs/AST-030/spec.md+++ b/docs/specs/AST-030/spec.md@@ -40,7 +40,10 @@ affects_consumer_docs: []".github/actions/test-scope/**",".github/scripts/workflow-paths.test.mjs",".github/actions/heavy-caps/**",- ".github/scripts/ci-pr-gate.*"+ ".github/scripts/ci-pr-gate.*",+ ".github/scripts/perf-*",+ "apps/sandbox/perf/**",+ "playwright.perf.config.ts"]}```@@ -355,6 +358,23 @@ they are separately named.required check would count as passing. Events other than a pull request (apush to `main`, a merge-group entry, a dispatch) are unaffected, and a fork'scode never runs with the repository's secrets.+- **FR18 — Runtime performance has one nightly lane, outside the pull-request+ surfaces.** `perf-nightly.yml` measures a fixed set of Sandbox perf scenes in+ Chromium on every nightly run of `main` and on manual dispatch: mount long+ tasks and layouts, interaction layouts and latency, and the JS heap and event+ listeners left after unmount, as the median of several samples. Its result is+ trend data: it reports no required context and gates no merge or release. A+ nightly or dispatched run on `main` is compared with the median of `main`'s+ recorded runs in the seven days before it; a metric more than 10% worse, and+ worse by at least that metric's floor, on that run and on the recorded run+ before it opens the one open regression issue or comments on it, naming the+ commits since the last run with no metric over the threshold. A metric over+ the threshold on one run alone is reported as pending. A pull request runs the+ lane only while it carries `ci:run-perf`, last applied by someone with write,+ maintain or admin access, under the same label rule and timing as FR12's suite+ labels and FR17, and only when its paths touch what the scenes measure (FR14);+ a pull request run is compared with the same history and records nothing. The+ lane runs on standard runners within the pull-request job budget.- **FR19 — A pull request that changes CI states its CI impact.** A pullrequest that changes a workflow, a composite action or a CI script carriesits CI impact in its description's `CI impact:` line: each changed job's@@ -432,6 +452,7 @@ the shared Storybook PR owner.Outside the classifier-routed workflows, the Internal Registry dependency check,the CLI smoke test and the hardening-issue opener run only when the paths theyread change (FR14).+The nightly perf lane is the one scheduled measurement outside them (FR18).## Verification@@ -450,6 +471,7 @@ read change (FR14).| FR15 | workflow contract and reconciler tests | ENGOWNER spec PR; owner command; other comment; fork review; non-spec PR; run over the request budget; maintainer dispatch | `review-required` succeeds while spec approval is missing, another workflow publishes a review context, a fork review starts the spec checks, a non-command comment starts a runner, or an over-budget run decides || FR16 | `heavy-caps-action.test.mjs` cap cases and heavy-job contract; `ci-admission.test.mjs` | sixteenth-oldest PR; 101st heavy run of the day, beside cancelled, light and other-branch runs; agent lane and its five-PR and forty-run backstops; an engineering owner, including the shared login past those backstops, a commented-out or trailing-comment handle and an unreadable roster; bot; the deciding run; re-run; failed lookup; a run with room, without room, past its longest wait, beside a waiting `main` check, as a draft or unlabelled fork, with opt-in labels, beside small workflows past their measured most, and with the API unreadable; each workflow's most jobs; every required context's rule | a heavy suite that runs ahead of the author's fifteen oldest PRs, or past the daily cap, a capped job that passes, an engineering owner or a re-run that is held, or a roster read from the pull request's head, a run that starts without room before its longest wait, a pull-request run that starts while `main` waits, a job admission could skip, or a peak below the jobs its run holds fails || FR17 | `ci-pr-gate.test.mjs` over every `pull_request` workflow and the shared conditions | branch here, fork with and without `ci:run`, another label, draft with the label, ready, push, merge group, dispatch | a job that can start on a pull request without the condition, a label trigger, a required job that skips for a pull request that may not run CI, or a draft or unlabelled fork that runs fails |+| FR18 | `perf-nightly-workflow.test.mjs`, `perf-trend.test.mjs` and `run-labels.test.mjs` | nightly and dispatch; labelled, unlabelled, draft and unvouched-fork pull requests; write, read and no access; newest label event; failed lookup; 10% boundary; metric floors; seven-day window edges; fewer than three runs; a metric over once, twice in a row, or a different metric before; recorded and pull request runs | a run on an unlabelled pull request, a label honoured from read access, a regression called at exactly 10%, under its floor or on one run alone, a run outside the window in the baseline, a pull request run that records, or a second regression issue fails || FR19 | `ci-impact.test.mjs`, `pr-preview.test.mjs` and the maintenance template | CI change and non-CI change; standard, non-standard and expression runners; added, removed and changed jobs; trigger change | a CI change reports no impact, a move onto a non-standard runner or a pool or time-to-green rise past its threshold goes unflagged, or the section names a price || FR22 | `main-validation-workflow.test.mjs` over the Main workflow and its commit resolver | six UI and six Node legs; dispatch without a commit, with a commit in main's history, with a commit outside it and with a malformed one; a re-check from the workflow token and from a person, and one that names `sha`; a `sha` before the size records, and one of only spaces | a hold, a leg past the fifteen-minute budget, a commit outside main's history that validates, a person's dispatch or a `sha` run that cancels or replaces a validation of main, a `sha` lane that checks out another commit, or a `sha` run named for main's tip fails || FR23 | `release-check-shape.test.mjs`, `a11y-contract-shard.test.mjs` and the release-check workflow contract | release-check and pull-request legs; a contract file longer than a leg's share, one it cannot name exactly, and a serial one; the recorded suite; a re-check from an admin, a maintainer and a writer, from another branch, with a short SHA and with an older commit; failed-only beside a passed, a running and a missing run | a narrower release-check, a held or over-budget leg, a test selected by no leg or by two, a re-check that a writer can start, that waits for main's running check, or that re-runs jobs of another commit fails |@@ -641,6 +663,26 @@ other writers read; it is not a required check, so a contributor waits onRejected: keeping a separate spec-owner workflow, which split one review stateacross two workflows and two visible gates.+### DEC-10 — Runtime performance is a nightly trend on main++**Reference:** `spec:AST-030/DEC-10`+**Decider:** `cixzhang`, `2026-10-31`++Runtime cost (long tasks, forced layouts, interaction latency, what an unmount+leaves behind) shows only in a real browser and drifts in small steps, so one+nightly measurement of fixed scenes on `main`, compared with its own recent+median, finds a regression within two days and names its commit range. A+standard runner's speed varies by about a fifth from one machine to the next,+more than the threshold, so a regression counts only when two recorded runs in a+row show it. It belongs to no surface's test owner: it reads composites across+components, it judges a trend rather than a pass or fail, and it reports no+required context, so FR11's rule against new lanes inside a surface does not+reach it. The history travels between recorded runs as a workflow artifact, so+the lane needs no write access to the repository's contents. A pull request opts+in with `ci:run-perf` on the same terms as the CI suite labels.++Rejected: measuring on every pull request (noise and runner time for a signal that needs a trend).+### DEC-11 — Main's release-check runs about once an hour**Reference:** `spec:AST-030/DEC-11`diff --git a/playwright.perf.config.ts b/playwright.perf.config.tsindex a5df7c9b1d..28fc078316 100644--- a/playwright.perf.config.ts+++ b/playwright.perf.config.ts@@ -4,7 +4,7 @@* @file playwright.perf.config.ts* @input Uses @playwright/test; reads the built Sandbox in apps/sandbox/out* @output The perf scenes, measured in desktop Chromium- * @position Run after+ * @position Run nightly by .github/workflows/perf-nightly.yml, after* `pnpm build && pnpm -F @astryxdesign/sandbox build`. Separate from* playwright.config.ts (accessibility contracts) and* playwright.interaction.config.ts (touch and animation proofs): a perf