An open source design system that's fully customizable and agent ready
TypeScript
ci: a pull request that changes CI shows its CI impact: CI workflow job-minutes, runner-pool usage and time to green, and runner classes of what changed (AST-030 FR19) (#9187)
Cindy Zhangb094f93bfea4
diff --git a/.github/PULL_REQUEST_TEMPLATE/maintenance.md b/.github/PULL_REQUEST_TEMPLATE/maintenance.mdindex aaff259181..661f6132f9 100644--- a/.github/PULL_REQUEST_TEMPLATE/maintenance.md+++ b/.github/PULL_REQUEST_TEMPLATE/maintenance.md@@ -17,6 +17,7 @@- Failure before the change:- Success after the change:- Product/runtime behavior verified unchanged:+- CI impact (for a change to workflows, actions or CI scripts): <!-- runner class of each changed job, job-minutes per update and runner-pool usage before and after, time to green; the PR report's CI impact section measures them -->## Scopediff --git a/.github/scripts/ci-free-runners.test.mjs b/.github/scripts/ci-free-runners.test.mjsindex a475357fd6..451a130d1e 100644--- a/.github/scripts/ci-free-runners.test.mjs+++ b/.github/scripts/ci-free-runners.test.mjs@@ -17,6 +17,9 @@ import fs from 'node:fs';import path from 'node:path';import {describe, expect, it} from 'vitest';import yaml from 'yaml';+import runnerLabels from './lib/runner-labels.cjs';++const {STANDARD_RUNNERS, labelsOf} = runnerLabels;const dir = path.resolve(import.meta.dirname, '../workflows');const workflows = fs@@ -28,18 +31,7 @@ const workflows = fs]);/** Standard hosted runners: free on a public repository. */-const FREE = new Set([- 'ubuntu-latest',- 'ubuntu-slim',- 'ubuntu-24.04',- 'ubuntu-22.04',- 'ubuntu-24.04-arm',- 'ubuntu-22.04-arm',- 'macos-latest',- 'macos-15',- 'macos-14',- 'windows-latest',-]);+const FREE = STANDARD_RUNNERS;/*** `workflow/job` → why that job may use a paid larger runner. Only the release@@ -71,48 +63,6 @@ const RELEASE_CHECK_JOBS = [const RELEASE_GATE ="github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && github.triggering_actor != 'github-actions[bot]'";-/**- * Every runner label a job can resolve to. Matrix values include `include`- * entries; an expression yields its quoted runner literals. Anything that does- * not resolve (`${{ vars.X }}`, `${{ inputs.X }}`, a computed matrix) is- * returned raw, so it fails the free-runner check instead of passing unseen.- */-function labelsOf(job) {- const runsOn = job['runs-on'];- if (runsOn == null) return [];- const labels = [];- for (const value of [runsOn].flat().map(String)) {- const matrix = /^\$\{\{\s*matrix\.([a-zA-Z0-9_-]+)\s*\}\}$/.exec(value);- if (matrix) {- const key = matrix[1];- const m = job.strategy?.matrix;- const found =- m && typeof m === 'object'- ? [- ...[m[key] ?? []].flat(),- ...[m.include ?? []].flat().map(entry => entry?.[key]),- // A computed include can add any runner: fail closed.- ...(typeof m.include === 'string' ? [value] : []),- ]- .filter(item => item != null)- .map(String)- : [];- labels.push(...(found.length > 0 ? found : [value]));- } else if (value.includes('${{')) {- // The one expression shape the workflows use, `cond && 'a' || 'b'`,- // resolves to both labels; any other expression stays raw and fails.- const pair =- /^\$\{\{\s*([^|]+?)\s*&&\s*'([^']+)'\s*\|\|\s*'([^']+)'\s*\}\}$/.exec(- value,- );- labels.push(...(pair ? [pair[2], pair[3]] : [value]));- } else {- labels.push(value);- }- }- return labels;-}-describe('runner labels', () => {it('finds runner labels to check', () => {const all = workflows.flatMap(([, wf]) =>diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.jsindex a514964138..314271a410 100644--- a/.github/scripts/generate-pr-comment.js+++ b/.github/scripts/generate-pr-comment.js@@ -3,7 +3,7 @@/*** @description Generates the PR comment from the run's analysis results- * @input --analysis <file> --base-stats <file> --package-size <file> --api-changes <file> --a11y <file> --visual <file> --visual-frames <file> --preview <file> --jobs <file> --touched <file> --source-conclusion <conclusion> --run-url <url>+ * @input --analysis <file> --base-stats <file> --package-size <file> --api-changes <file> --a11y <file> --visual <file> --visual-frames <file> --preview <file> --jobs <file> --touched <file> --ci-impact <file> --source-conclusion <conclusion> --run-url <url>* @output Formatted markdown comment body to stdout, compact by design: this* head's preview first, the modified components (preview links, size, lines* of code, a mark per browser suite that ran; three rows visible, the rest@@ -16,6 +16,7 @@const fs = require('node:fs');const {buildA11ySection} = require('./lib/a11y-format');const {buildCompactReport} = require('./lib/pr-status');+const {renderCiImpact} = require('./lib/ci-impact');// Report-file strings render as literal inline text, report numbers as// numbers — the comment's structure comes only from this file's literals.const {inline, num} = require('./lib/report-text');@@ -71,6 +72,8 @@ const packageSize = readOptionalJson('package-size');const visualFrames = readOptionalJson('visual-frames');// The API report step's changes for this head, when it ran.const apiChanges = readOptionalJson('api-changes');+// A CI change's impact on runner usage and time to green (ci-impact.js).+const ciImpact = readOptionalJson('ci-impact');const visualVerdict = readOptionalJson('visual');let a11yReport = null;try {@@ -141,7 +144,7 @@ const footerLinksStr = runUrlconst body = `## PR Analysis Report<!-- astryx-pr-analysis -->-${report || '_No component or package changed._\n\n'}${conclusionNote}${diffModeCaveat}${visualFailed}${a11yDetails}---+${report || '_No component or package changed._\n\n'}${renderCiImpact(ciImpact)}${conclusionNote}${diffModeCaveat}${visualFailed}${a11yDetails}---<sub>Generated by PR Enrichment workflow${footerLinksStr}</sub>`;diff --git a/.github/scripts/lib/ci-impact.js b/.github/scripts/lib/ci-impact.jsnew file mode 100644index 0000000000..4d7fe2b871--- /dev/null+++ b/.github/scripts/lib/ci-impact.js@@ -0,0 +1,506 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file ci-impact.js+ * @input A pull request's changed files, CI's workflow file at the merge base+ * and at the head, the source run's jobs, and a sample of recent pull-request+ * CI runs (their jobs and wall-clock times, plus how many ran in 7 days)+ * @output For a pull request that changes CI: each changed job's runner class+ * (standard GitHub-hosted, or non-standard), job-minutes per pull-request+ * update before and after, the change in runner-pool usage (runner-minutes+ * a day and average concurrent jobs), time to green, runs a day, the flags+ * that need a reviewer's eye, and that as one collapsed report section+ * @position Used by the PR comment job (pr-preview.mjs gathers the data,+ * generate-pr-comment.js renders it). It reads only run history the job can+ * already see; nothing here starts a job. It reports runner usage, never+ * prices.+ */++const {inline, num} = require('./report-text');+const {baseJobName} = require('./pr-status');+const {+ STANDARD_RUNNERS,+ labelsOf,+ isUnresolved,+} = require('./runner-labels.cjs');++/** Paths whose change makes a pull request a CI change. */+const CI_PATH =+ /^(\.github\/workflows\/|\.github\/actions\/|\.github\/scripts\/|\.github\/ci-|scripts\/ci-)/;++const FLAG_POOL_PERCENT = 10;+const FLAG_WALL_MINUTES = 2;++function isCiChange(files) {+ return (files || []).some(file => CI_PATH.test(file));+}++/**+ * A runner class from the labels a job can resolve to (runner-labels.cjs):+ * `standard`, `non-standard`, `mixed` (an expression that picks either), or+ * `unknown` (a label no reading can resolve, such as `${{ vars.X }}`).+ */+function classifyLabels(all) {+ const labels = [...new Set(all.map(String))];+ const unresolved = labels.filter(isUnresolved);+ const resolved = labels.filter(label => !isUnresolved(label));+ const nonStandard = resolved.filter(label => !STANDARD_RUNNERS.has(label));+ if (labels.length === 0 || unresolved.length > 0)+ return {kind: 'unknown', labels, nonStandard};+ if (nonStandard.length === 0) return {kind: 'standard', labels};+ if (nonStandard.length === resolved.length)+ return {kind: 'non-standard', labels};+ return {kind: 'mixed', labels, nonStandard};+}++/** A job's runner class: from the job, or from a bare `runs-on` value. */+function runnerClass(jobOrRunsOn) {+ const job =+ jobOrRunsOn &&+ typeof jobOrRunsOn === 'object' &&+ !Array.isArray(jobOrRunsOn)+ ? jobOrRunsOn+ : {'runs-on': jobOrRunsOn};+ return classifyLabels(labelsOf(job));+}++/**+ * One class for jobs that share a name: every label any of them can pick. A+ * variant that always runs on a non-standard runner makes the job+ * non-standard, whatever the other variants use.+ */+function mergeClasses(a, b) {+ if (!a) return b;+ if (!b) return a;+ const merged = classifyLabels([...a.labels, ...b.labels]);+ if (a.kind === 'non-standard' || b.kind === 'non-standard')+ return {kind: 'non-standard', labels: merged.labels};+ return merged;+}++/** A job's name as the run shows it, without its matrix suffix. */+function displayName(id, job) {+ return baseJobName(job?.name ?? id);+}++/** The non-standard labels a runner class can pick. */+function nonStandardLabels(cls) {+ if (!cls) return [];+ if (cls.kind === 'non-standard') return cls.labels;+ return cls.nonStandard ?? [];+}++/** Whether a job's head runner adds a non-standard label its base did not have. */+function movesOntoNonStandard(before, after) {+ const was = nonStandardLabels(before);+ return nonStandardLabels(after).some(label => !was.includes(label));+}++/**+ * Job by job, what changed between two parsed workflow files: added, removed,+ * or changed (any part of its definition), with its runner class on each side.+ */+function diffWorkflow(base, head) {+ const baseJobs = base?.jobs || {};+ const headJobs = head?.jobs || {};+ const ids = [+ ...new Set([...Object.keys(baseJobs), ...Object.keys(headJobs)]),+ ].sort();+ const changes = [];+ for (const id of ids) {+ const before = baseJobs[id];+ const after = headJobs[id];+ if (before && after && JSON.stringify(before) === JSON.stringify(after))+ continue;+ changes.push({+ id,+ name: displayName(id, after ?? before),+ // The base side's name: a renamed job's history is under it.+ baseName: before ? displayName(id, before) : null,+ change: !before ? 'added' : !after ? 'removed' : 'changed',+ before: before ? runnerClass(before) : null,+ after: after ? runnerClass(after) : null,+ });+ }+ // Decided per job id, before variants share a name below.+ const movedOntoNonStandard = [+ ...new Set(+ changes+ .filter(change => movesOntoNonStandard(change.before, change.after))+ .map(change => change.name),+ ),+ ];+ // Variants of one job (`pr-a11y (browser specs)`, `pr-a11y (full audit)`)+ // share a name in the report: one entry per name and kind of change, with+ // the runner classes of all of them merged.+ const byKey = new Map();+ for (const change of changes) {+ const key = `${change.name}:${change.change}`;+ const seen = byKey.get(key);+ byKey.set(+ key,+ seen+ ? {+ ...seen,+ before: mergeClasses(seen.before, change.before),+ after: mergeClasses(seen.after, change.after),+ }+ : change,+ );+ }+ const unique = [...byKey.values()];+ return {+ changes: unique,+ movedOntoNonStandard,+ triggersChanged: JSON.stringify(base?.on) !== JSON.stringify(head?.on),+ names: {+ before: Object.entries(baseJobs).map(([id, job]) => displayName(id, job)),+ after: Object.entries(headJobs).map(([id, job]) => displayName(id, job)),+ },+ };+}++function minutes(job) {+ const start = Date.parse(job.started_at ?? job.startedAt ?? '');+ const end = Date.parse(job.completed_at ?? job.completedAt ?? '');+ return Number.isFinite(start) && Number.isFinite(end) && end > start+ ? (end - start) / 60000+ : 0;+}++function median(values) {+ if (values.length === 0) return null;+ const sorted = [...values].sort((a, b) => a - b);+ const middle = Math.floor(sorted.length / 2);+ return sorted.length % 2+ ? sorted[middle]+ : (sorted[middle - 1] + sorted[middle]) / 2;+}++/**+ * From a sample of recent pull-request runs: each job's average runner+ * minutes per run (its legs summed, 0 when it did not run), the median+ * wall-clock time to a finished run, and runs a day.+ */+function summarizeHistory({runs, runsLast7Days}) {+ const perJob = new Map();+ const wall = [];+ // Each run's set of jobs that did work, beside its wall-clock time, so time+ // to green compares only with runs that ran the same jobs.+ const wallBySet = [];+ for (const run of runs || []) {+ const totals = new Map();+ for (const job of run.jobs || []) {+ if (job.conclusion === 'skipped') continue;+ const name = displayName(job.name, job);+ totals.set(name, (totals.get(name) ?? 0) + minutes(job));+ }+ for (const [name, value] of totals) {+ if (!perJob.has(name)) perJob.set(name, 0);+ perJob.set(name, perJob.get(name) + value);+ }+ const started = Date.parse(run.run_started_at ?? run.created_at ?? '');+ const finished = Date.parse(run.updated_at ?? '');+ if (+ Number.isFinite(started) &&+ Number.isFinite(finished) &&+ finished > started+ ) {+ wall.push((finished - started) / 60000);+ wallBySet.push({+ jobs: [...totals.keys()].sort().join('\n'),+ minutes: (finished - started) / 60000,+ });+ }+ }+ const count = Math.max(1, (runs || []).length);+ // How many sampled runs each job did work in.+ const jobRuns = {};+ for (const run of runs || []) {+ const ran = new Set(+ (run.jobs || [])+ .filter(job => job.conclusion !== 'skipped')+ .map(job => displayName(job.name, job)),+ );+ for (const name of ran) jobRuns[name] = (jobRuns[name] ?? 0) + 1;+ }+ return {+ sampled: (runs || []).length,+ jobMinutes: Object.fromEntries(+ [...perJob].map(([name, total]) => [name, total / count]),+ ),+ jobRuns,+ wallMinutes: median(wall),+ wallBySet,+ runsPerDay: num(runsLast7Days) / 7,+ };+}++/** Each job's runner minutes in this run (legs summed), by display name. */+function runMinutes(jobs) {+ const totals = {};+ for (const job of jobs || []) {+ if (job.conclusion === 'skipped') continue;+ const name = displayName(job.name, job);+ totals[name] = (totals[name] ?? 0) + minutes(job);+ }+ return totals;+}++/**+ * Before and after for one pull-request update. Before is each base job's+ * recent average; after keeps that average for an unchanged job and uses this+ * run's own minutes for an added or changed one (a removed job counts zero).+ */+function estimateImpact({diff, history, currentJobs, currentWallMinutes}) {+ const current = runMinutes(currentJobs);+ // A changed job counts this run's minutes only as often as it ran in the+ // sampled runs (a label-gated suite tested with its label rarely runs, and+ // one that ran in none counts zero); an added job, which has no history, as+ // often as it ran here (once).+ const added = new Set(+ diff.changes.filter(c => c.change === 'added').map(c => c.name),+ );+ // A changed job's history is under its base-side name (a rename keeps it).+ const baseNameOf = new Map(+ diff.changes+ .filter(c => c.change === 'changed' && c.baseName)+ .map(c => [c.name, c.baseName]),+ );+ const historyName = name => baseNameOf.get(name) ?? name;+ const share = name =>+ added.has(name) || history.sampled === 0+ ? 1+ : (history.jobRuns?.[historyName(name)] ?? 0) / history.sampled;+ const changed = new Set(+ diff.changes.filter(c => c.change !== 'removed').map(c => c.name),+ );+ const sum = (names, pick) =>+ [...new Set(names)].reduce((total, name) => total + pick(name), 0);+ const before = sum(diff.names.before, name => history.jobMinutes[name] ?? 0);+ const after = sum(diff.names.after, name =>+ changed.has(name) && current[name] !== undefined+ ? current[name] * share(name)+ : (history.jobMinutes[historyName(name)] ?? 0),+ );+ const perDay = value => value * history.runsPerDay;+ const poolBefore = perDay(before);+ const poolAfter = perDay(after);+ const poolPercent =+ poolBefore > 0 ? ((poolAfter - poolBefore) / poolBefore) * 100 : 0;+ // Time to green compares with the median of sampled runs that ran the+ // same jobs as this run; with none, it is not compared.+ const ranHere = Object.keys(current).sort().join('\n');+ const alike = (history.wallBySet || [])+ .filter(entry => entry.jobs === ranHere)+ .map(entry => entry.minutes);+ const wallBefore = alike.length > 0 ? median(alike) : null;+ const wallDelta =+ wallBefore !== null && Number.isFinite(currentWallMinutes)+ ? currentWallMinutes - wallBefore+ : null;+ const flags = [+ ...(diff.movedOntoNonStandard || []).map(+ name => `${inline(name)} moves onto a non-standard runner`,+ ),+ ...(poolPercent > FLAG_POOL_PERCENT+ ? [`runner-pool usage +${poolPercent.toFixed(0)}%`]+ : []),+ ...(wallDelta !== null && wallDelta > FLAG_WALL_MINUTES+ ? [`time to green +${wallDelta.toFixed(1)} min`]+ : []),+ ...(diff.triggersChanged+ ? ['the workflow triggers changed, so runs a day may change too']+ : []),+ ];+ return {+ jobMinutes: {before, after},+ pool: {before: poolBefore, after: poolAfter, percent: poolPercent},+ concurrent: {before: poolBefore / 1440, after: poolAfter / 1440},+ wall: {+ before: wallBefore,+ after: currentWallMinutes ?? null,+ delta: wallDelta,+ },+ runsPerDay: history.runsPerDay,+ sampled: history.sampled,+ changes: diff.changes,+ flags,+ };+}++/** When a workflow runs, in words: its events, and a schedule's cron times. */+function triggerText(on) {+ if (!on) return 'no trigger';+ if (typeof on === 'string') return inline(on);+ if (Array.isArray(on)) return on.map(inline).join(', ');+ return Object.entries(on)+ .map(([event, config]) =>+ event === 'schedule' && Array.isArray(config)+ ? `schedule (${config.map(entry => inline(entry.cron)).join(', ')})`+ : inline(event),+ )+ .join(', ');+}++/**+ * A changed workflow other than CI's own: whether it was added, removed or+ * changed, when it runs, and each job's runner class, compared job by job.+ * Its minutes are not measured here: this run's history covers CI's own+ * workflow.+ */+function describeWorkflow(file, base, head) {+ const doc = head ?? base;+ const jobs = Object.entries(doc?.jobs || {}).map(([id, job]) => ({+ name: displayName(id, job),+ runner: runnerClass(job),+ }));+ const baseJobs = base?.jobs || {};+ return {+ file,+ change: !base ? 'added' : !head ? 'removed' : 'changed',+ triggers: triggerText(doc?.on),+ jobs,+ nonStandard: Object.entries(head?.jobs || {}).some(([id, job]) =>+ movesOntoNonStandard(+ baseJobs[id] ? runnerClass(baseJobs[id]) : null,+ runnerClass(job),+ ),+ ),+ };+}++function signed(value, digits = 0, unit = '') {+ const rounded = Number(value.toFixed(digits));+ if (rounded === 0) return `0${unit}`;+ return `${rounded > 0 ? '+' : '−'}${Math.abs(rounded).toLocaleString('en-US', {maximumFractionDigits: digits})}${unit}`;+}++function amount(value, digits = 0) {+ return Number(value.toFixed(digits)).toLocaleString('en-US', {+ maximumFractionDigits: digits,+ });+}++function runnerText(cls) {+ if (!cls) return '—';+ if (cls.kind === 'standard') return 'standard';+ if (cls.kind === 'unknown')+ return 'unknown (an expression this report cannot resolve)';+ if (cls.kind === 'non-standard')+ return `non-standard (${cls.labels.map(inline).join(', ')})`;+ return `standard or non-standard by event (${cls.nonStandard.map(inline).join(', ')})`;+}++/** The runner classes of what changed, in a few words. */+function runnerSummary(classes) {+ if (classes.some(cls => cls.kind === 'non-standard'))+ return 'non-standard runners';+ if (classes.some(cls => cls.kind === 'mixed' || cls.kind === 'unknown'))+ return 'some runners chosen by event or unknown';+ return 'standard runners';+}++/** The report section: a visible summary line, the table collapsed under it. */+function renderCiImpact(impact) {+ if (!impact) return '';+ if (impact.unavailable)+ return '**CI impact:** not measured on this run; the run history or the workflow files could not be read.\n\n';+ const {jobMinutes, pool, concurrent, wall} = impact;+ // A pool change under 10% keeps one decimal, so +0.5% does not read as 0%.+ const percent = value =>+ value === 0 ? '0%' : `${signed(value, Math.abs(value) < 10 ? 1 : 0)}%`;+ const summary = [+ `${signed(jobMinutes.after - jobMinutes.before, 1)} job-min per update`,+ `pool ${percent(pool.percent)}`,+ wall.delta === null+ ? null+ : `time to green ${signed(wall.delta, 1, ' min')}`,+ runnerSummary([+ ...impact.changes.map(change => change.after).filter(Boolean),+ ...(impact.otherWorkflows || []).flatMap(workflow =>+ workflow.change === 'removed'+ ? []+ : workflow.jobs.map(job => job.runner),+ ),+ ]),+ (impact.otherWorkflows || []).length > 0+ ? `${(impact.otherWorkflows || []).length} other workflow${(impact.otherWorkflows || []).length === 1 ? '' : 's'} changed (not measured)`+ : null,+ (impact.unmeasuredFiles || []).length > 0+ ? `${impact.unmeasuredFiles.length} CI action or script file${impact.unmeasuredFiles.length === 1 ? '' : 's'} changed (not measured)`+ : null,+ ].filter(Boolean);+ const rows = [+ [+ 'Job-minutes per PR update',+ amount(jobMinutes.before, 1),+ amount(jobMinutes.after, 1),+ signed(jobMinutes.after - jobMinutes.before, 1),+ ],+ [+ 'Runner pool, runner-minutes a day',+ amount(pool.before),+ amount(pool.after),+ `${signed(pool.after - pool.before)} (${percent(pool.percent)})`,+ ],+ [+ 'Average concurrent jobs',+ amount(concurrent.before, 1),+ amount(concurrent.after, 1),+ signed(concurrent.after - concurrent.before, 1),+ ],+ [+ 'Time to green (median → this run)',+ wall.before === null ? '—' : `${amount(wall.before, 1)} min`,+ wall.after === null ? '—' : `${amount(wall.after, 1)} min`,+ wall.delta === null ? '—' : signed(wall.delta, 1, ' min'),+ ],+ ['Runs a day', amount(impact.runsPerDay), amount(impact.runsPerDay), '0'],+ ];+ const workflows = (impact.otherWorkflows || [])+ .map(+ workflow =>+ `${inline(workflow.file)} ${workflow.change}: runs on ${workflow.triggers}; ${+ workflow.jobs.length === 0+ ? 'no jobs'+ : [+ ...new Set(workflow.jobs.map(job => runnerText(job.runner))),+ ].join(', ')+ } runners${workflow.change === 'removed' ? '' : '; its minutes are not measured here'}`,+ )+ .join(' · ');+ const files = impact.unmeasuredFiles || [];+ const unmeasured =+ files.length > 0+ ? `Not measured (the jobs that use these keep their recent minutes above): ${files+ .slice(0, 10)+ .map(inline)+ .join(+ ', ',+ )}${files.length > 10 ? `, and ${files.length - 10} more` : ''}.\n\n`+ : '';+ const jobs = impact.changes+ .map(+ change =>+ `${inline(change.name)} ${change.change}: ${change.change === 'removed' ? runnerText(change.before) : runnerText(change.after)}`,+ )+ .join(' · ');+ return `<details>\n<summary>CI impact: ${summary.join(' · ')}${impact.flags.length > 0 ? ' ⚠️' : ''}</summary>\n\n| | Before | After | Change |\n|---|---|---|---|\n${rows+ .map(row => `| ${row.join(' | ')} |`)+ .join(+ '\n',+ )}\n\n${jobs ? `Changed jobs: ${jobs}\n\n` : ''}${workflows ? `Other workflows: ${workflows}\n\n` : ''}${unmeasured}${impact.flags.length > 0 ? `**Flags:** ${impact.flags.join('; ')}.\n\n` : ''}Assumptions: before is each job's average over the last ${num(impact.sampled)} pull-request runs; after keeps that for unchanged jobs and uses this run's minutes for added or changed ones, weighted by how often each changed job ran in those runs; runs a day averages the seven whole days before today; time to green compares this run with the median of those runs that ran the same jobs.\n\n</details>\n\n`;+}++module.exports = {+ inlineName: inline,+ describeWorkflow,+ CI_PATH,+ isCiChange,+ runnerClass,+ diffWorkflow,+ summarizeHistory,+ estimateImpact,+ renderCiImpact,+};diff --git a/.github/scripts/lib/ci-impact.test.mjs b/.github/scripts/lib/ci-impact.test.mjsnew file mode 100644index 0000000000..41457135a2--- /dev/null+++ b/.github/scripts/lib/ci-impact.test.mjs@@ -0,0 +1,535 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file ci-impact.test.mjs+ * Pins the CI-impact section: which pull requests count as CI changes, the+ * runner class of a `runs-on` value, the job-by-job workflow diff, the+ * before/after estimate (unchanged jobs keep their history, added or changed+ * jobs use this run), the flags, and a section with no prices in it.+ */++import {describe, expect, it} from 'vitest';+import impactLib from './ci-impact.js';++const {+ describeWorkflow,+ isCiChange,+ runnerClass,+ diffWorkflow,+ summarizeHistory,+ estimateImpact,+ renderCiImpact,+} = impactLib;++const at = minute => new Date(Date.UTC(2026, 9, 31, 12, minute)).toISOString();+const job = (name, start, end, conclusion = 'success') => ({+ name,+ conclusion,+ started_at: at(start),+ completed_at: at(end),+});++describe('isCiChange', () => {+ it('counts workflows, actions and CI scripts', () => {+ expect(isCiChange(['.github/workflows/ci.yml'])).toBe(true);+ expect(isCiChange(['.github/actions/setup/action.yml'])).toBe(true);+ expect(isCiChange(['.github/scripts/lib/pr-status.js'])).toBe(true);+ expect(isCiChange(['packages/core/src/Button/Button.tsx'])).toBe(false);+ });+});++describe('runnerClass', () => {+ it.each([+ ['ubuntu-latest', 'standard'],+ ['ubuntu-slim', 'standard'],+ ['ubuntu-24.04-arm', 'standard'],+ ['macos-15', 'standard'],+ ['4-core-ubuntu', 'non-standard'],+ ["${{ inputs.release && '4-core-ubuntu' || 'ubuntu-latest' }}", 'mixed'],+ ])('reads %s as %s', (runsOn, kind) => {+ expect(runnerClass(runsOn).kind).toBe(kind);+ });+});++describe('runnerClass for an expression without labels', () => {+ it('is unknown, never standard', () => {+ expect(runnerClass('${{ matrix.os }}').kind).toBe('unknown');+ });+});++describe('diffWorkflow', () => {+ it('lists added, removed and changed jobs with their runner class', () => {+ const base = {+ on: {pull_request: {}},+ jobs: {+ lint: {'runs-on': 'ubuntu-latest', steps: [1]},+ old: {'runs-on': 'ubuntu-latest'},+ build: {'runs-on': 'ubuntu-latest', steps: [1]},+ },+ };+ const head = {+ on: {pull_request: {}},+ jobs: {+ lint: {'runs-on': 'ubuntu-latest', steps: [1]},+ build: {'runs-on': '4-core-ubuntu', steps: [1]},+ fresh: {name: 'Fresh check', 'runs-on': 'ubuntu-slim'},+ },+ };+ const diff = diffWorkflow(base, head);+ expect(diff.changes.map(change => [change.name, change.change])).toEqual([+ ['build', 'changed'],+ ['Fresh check', 'added'],+ ['old', 'removed'],+ ]);+ expect(diff.triggersChanged).toBe(false);+ });+});++describe('describeWorkflow', () => {+ it('says when an added workflow runs and on which runners', () => {+ const workflow = describeWorkflow('perf-nightly.yml', null, {+ on: {schedule: [{cron: '17 9 * * *'}], workflow_dispatch: {}},+ jobs: {perf: {'runs-on': 'ubuntu-latest'}},+ });+ expect(workflow).toMatchObject({+ file: 'perf-nightly.yml',+ change: 'added',+ triggers: 'schedule (17 9 \\* \\* \\*), workflow\\_dispatch',+ nonStandard: false,+ });+ });++ it('flags a workflow that moves onto a non-standard runner', () => {+ expect(+ describeWorkflow(+ 'nightly.yml',+ {on: 'push', jobs: {a: {'runs-on': 'ubuntu-latest'}}},+ {on: 'push', jobs: {a: {'runs-on': '4-core-ubuntu'}}},+ ).nonStandard,+ ).toBe(true);+ });+});++describe('runner classes shared with the free-runner contract', () => {+ it('reads a variable with a default as unknown, never standard', () => {+ expect(+ runnerClass("${{ vars.LARGE_RUNNER || 'ubuntu-latest' }}").kind,+ ).toBe('unknown');+ });++ it('merges the classes of jobs that share a name', () => {+ const diff = diffWorkflow(+ {+ on: 'pull_request',+ jobs: {+ a: {name: 'pr-a11y (browser)', 'runs-on': 'ubuntu-latest'},+ b: {name: 'pr-a11y (full)', 'runs-on': 'ubuntu-latest'},+ },+ },+ {+ on: 'pull_request',+ jobs: {+ a: {+ name: 'pr-a11y (browser)',+ 'runs-on': 'ubuntu-latest',+ steps: [1],+ },+ b: {+ name: 'pr-a11y (full)',+ 'runs-on':+ "${{ inputs.release && '2-core-ubuntu-arm' || 'ubuntu-24.04-arm' }}",+ },+ },+ },+ );+ expect(diff.changes).toHaveLength(1);+ expect(diff.changes[0].after.kind).toBe('mixed');+ expect(diff.changes[0].after.nonStandard).toEqual(['2-core-ubuntu-arm']);+ });+});++describe('a variant that always uses a larger runner', () => {+ it('makes the merged job non-standard, in the rendered line too', () => {+ const diff = diffWorkflow(+ {+ on: 'pull_request',+ jobs: {+ a: {name: 'pr-rtl (core)', 'runs-on': 'ubuntu-latest'},+ b: {name: 'pr-rtl (lab)', 'runs-on': 'ubuntu-latest'},+ },+ },+ {+ on: 'pull_request',+ jobs: {+ a: {name: 'pr-rtl (core)', 'runs-on': 'ubuntu-latest', steps: [1]},+ b: {name: 'pr-rtl (lab)', 'runs-on': '4-core-ubuntu'},+ },+ },+ );+ expect(diff.changes[0].after.kind).toBe('non-standard');+ const section = renderCiImpact({+ jobMinutes: {before: 1, after: 1},+ pool: {before: 10, after: 10, percent: 0},+ concurrent: {before: 0, after: 0},+ wall: {before: null, after: null, delta: null},+ runsPerDay: 10,+ sampled: 1,+ changes: diff.changes,+ flags: [],+ });+ expect(section).toContain(+ 'Changed jobs: pr-rtl changed: non-standard (ubuntu-latest, 4-core-ubuntu)',+ );+ expect(section).toContain('· non-standard runners</summary>');+ });+});++describe('non-standard moves', () => {+ it('are decided per job id, before variants share a name', () => {+ const diff = diffWorkflow(+ {+ on: 'pull_request',+ jobs: {+ a11y_browser: {name: 'pr-a11y (browser)', 'runs-on': 'ubuntu-latest'},+ a11y_full: {name: 'pr-a11y (full)', 'runs-on': 'ubuntu-latest'},+ },+ },+ {+ on: 'pull_request',+ jobs: {+ a11y_browser: {+ name: 'pr-a11y (browser)',+ 'runs-on': 'ubuntu-latest',+ steps: [1],+ },+ a11y_full: {name: 'pr-a11y (full)', 'runs-on': '4-core-ubuntu'},+ },+ },+ );+ expect(diff.changes).toHaveLength(1);+ expect(diff.movedOntoNonStandard).toEqual(['pr-a11y']);+ });++ it('compare another workflow job by job', () => {+ const base = {+ on: 'schedule',+ jobs: {a: {'runs-on': '4-core-ubuntu'}, b: {'runs-on': 'ubuntu-latest'}},+ };+ const head = {+ on: 'schedule',+ jobs: {a: {'runs-on': '4-core-ubuntu'}, b: {'runs-on': '4-core-ubuntu'}},+ };+ expect(describeWorkflow('nightly.yml', base, head).nonStandard).toBe(true);+ expect(describeWorkflow('nightly.yml', base, base).nonStandard).toBe(false);+ });+});++describe('estimateImpact', () => {+ const history = summarizeHistory({+ runsLast7Days: 70,+ runs: [+ {+ run_started_at: at(0),+ updated_at: at(10),+ jobs: [+ job('lint', 0, 2),+ job('build (1/2)', 0, 4),+ job('build (2/2)', 0, 4),+ job('old', 0, 1),+ ],+ },+ {+ run_started_at: at(0),+ updated_at: at(12),+ jobs: [+ job('lint', 0, 2),+ job('build (1/2)', 0, 4),+ job('build (2/2)', 0, 4),+ job('old', 0, 1, 'skipped'),+ ],+ },+ ],+ });++ it('keeps unchanged jobs at their history and uses this run for changed ones', () => {+ expect(history.jobMinutes).toEqual({lint: 2, build: 8, old: 0.5});+ expect(history.runsPerDay).toBe(10);+ const impact = estimateImpact({+ diff: {+ changes: [+ {+ name: 'build',+ change: 'changed',+ before: runnerClass('ubuntu-latest'),+ after: runnerClass('ubuntu-latest'),+ },+ {+ name: 'old',+ change: 'removed',+ before: runnerClass('ubuntu-latest'),+ after: null,+ },+ ],+ triggersChanged: false,+ names: {before: ['lint', 'build', 'old'], after: ['lint', 'build']},+ },+ history,+ currentJobs: [+ job('lint', 0, 9),+ job('build (1/2)', 0, 5),+ job('build (2/2)', 0, 5),+ ],+ currentWallMinutes: 11.5,+ });+ // lint keeps its 2-minute history despite this run's 9; build uses this run's 10.+ expect(impact.jobMinutes).toEqual({before: 10.5, after: 12});+ expect(impact.pool.before).toBe(105);+ expect(impact.pool.after).toBe(120);+ // Time to green compares with the one sampled run that ran lint and build only.+ expect(impact.wall).toEqual({before: 12, after: 11.5, delta: -0.5});+ expect(impact.flags).toEqual(['runner-pool usage +14%']);+ });++ it('flags a move onto a non-standard runner, a slower time to green, and new triggers', () => {+ const impact = estimateImpact({+ diff: {+ changes: [+ {+ name: 'build',+ change: 'changed',+ before: runnerClass('ubuntu-latest'),+ after: runnerClass('4-core-ubuntu'),+ },+ ],+ triggersChanged: true,+ movedOntoNonStandard: ['build'],+ names: {before: ['build'], after: ['build']},+ },+ history,+ currentJobs: [job('lint', 0, 2), job('build', 0, 8)],+ currentWallMinutes: 15,+ });+ expect(impact.flags).toEqual([+ 'build moves onto a non-standard runner',+ 'time to green +3.0 min',+ 'the workflow triggers changed, so runs a day may change too',+ ]);+ });+});++describe('a label-gated suite tested with its label', () => {+ // Ten sampled runs; the visual suite ran in one of them (its label).+ const runs = Array.from({length: 10}, (_, i) => ({+ run_started_at: at(0),+ updated_at: at(i === 0 ? 20 : 10),+ jobs: [+ job('lint', 0, 2),+ job('visual', 0, 10, i === 0 ? 'success' : 'skipped'),+ ],+ }));+ const history = summarizeHistory({runsLast7Days: 70, runs});++ it('weights the changed suite by how often it ran, and compares time to green with alike runs', () => {+ const impact = estimateImpact({+ diff: {+ changes: [+ {+ name: 'visual',+ change: 'changed',+ before: runnerClass('ubuntu-latest'),+ after: runnerClass('ubuntu-latest'),+ },+ ],+ triggersChanged: false,+ names: {before: ['lint', 'visual'], after: ['lint', 'visual']},+ },+ history,+ // This run had the label: the suite ran, a little slower.+ currentJobs: [job('lint', 0, 2), job('visual', 0, 11)],+ currentWallMinutes: 21,+ });+ // Before: lint 2 + visual 10 × 1/10; after: lint 2 + visual 11 × 1/10.+ expect(impact.jobMinutes.before).toBeCloseTo(3);+ expect(impact.jobMinutes.after).toBeCloseTo(3.1);+ expect(impact.wall).toEqual({before: 20, after: 21, delta: 1});+ expect(impact.flags).toEqual([]);+ });+});++describe('a changed job that ran in none of the sampled runs', () => {+ it('counts zero, not every later pull request', () => {+ const history = summarizeHistory({+ runsLast7Days: 70,+ runs: [+ {+ run_started_at: at(0),+ updated_at: at(10),+ jobs: [job('lint', 0, 2), job('perf', 0, 9, 'skipped')],+ },+ ],+ });+ const impact = estimateImpact({+ diff: {+ changes: [+ {+ name: 'perf',+ change: 'changed',+ before: runnerClass('ubuntu-latest'),+ after: runnerClass('ubuntu-latest'),+ },+ ],+ triggersChanged: false,+ names: {before: ['lint', 'perf'], after: ['lint', 'perf']},+ },+ history,+ currentJobs: [job('lint', 0, 2), job('perf', 0, 9)],+ currentWallMinutes: 11,+ });+ expect(impact.jobMinutes).toEqual({before: 2, after: 2});+ });+});++describe('a renamed job', () => {+ it('keeps its history under the base-side name', () => {+ const history = summarizeHistory({+ runsLast7Days: 70,+ runs: [+ {+ run_started_at: at(0),+ updated_at: at(10),+ jobs: [job('build', 0, 10)],+ },+ ],+ });+ const diff = diffWorkflow(+ {+ on: 'pull_request',+ jobs: {b: {name: 'build', 'runs-on': 'ubuntu-latest'}},+ },+ {+ on: 'pull_request',+ jobs: {b: {name: 'compile', 'runs-on': 'ubuntu-latest', steps: [1]}},+ },+ );+ expect(diff.changes[0]).toMatchObject({+ name: 'compile',+ baseName: 'build',+ change: 'changed',+ });+ const impact = estimateImpact({+ diff,+ history,+ currentJobs: [job('compile', 0, 14)],+ currentWallMinutes: 14,+ });+ expect(impact.jobMinutes).toEqual({before: 10, after: 14});+ expect(impact.pool.percent).toBeCloseTo(40);+ expect(impact.flags).toContain('runner-pool usage +40%');+ });+});++describe('renderCiImpact', () => {+ const impact = {+ jobMinutes: {before: 52, after: 53},+ pool: {before: 4160, after: 4240, percent: 1.9},+ concurrent: {before: 2.9, after: 2.94},+ wall: {before: 14.2, after: 15, delta: 0.8},+ runsPerDay: 80,+ sampled: 20,+ changes: [+ {+ name: 'size-base',+ change: 'removed',+ before: runnerClass('ubuntu-latest'),+ after: null,+ },+ ],+ flags: [],+ };++ it('collapses the table under a one-line summary, and names no price', () => {+ const section = renderCiImpact(impact);+ expect(section).toMatch(+ /^<details>\n<summary>CI impact: \+1 job-min per update · pool \+1\.9% · time to green \+0\.8 min · standard runners<\/summary>/,+ );+ expect(section).toContain(+ '| Runner pool, runner-minutes a day | 4,160 | 4,240 | +80 (+1.9%) |',+ );+ expect(section).toContain('Changed jobs: size-base removed: standard');+ expect(section).not.toMatch(/\$|cost|price|budget/i);+ });++ it('marks a section with flags', () => {+ const section = renderCiImpact({+ ...impact,+ flags: ['runner-pool usage +14%'],+ });+ expect(section).toContain(' ⚠️</summary>');+ expect(section).toContain('**Flags:** runner-pool usage +14%.');+ });++ it('lists other changed workflows and counts added ones in the summary', () => {+ const section = renderCiImpact({+ ...impact,+ otherWorkflows: [+ describeWorkflow('perf-nightly.yml', null, {+ on: {schedule: [{cron: '17 9 * * *'}]},+ jobs: {perf: {'runs-on': 'ubuntu-latest'}},+ }),+ ],+ });+ expect(section).toContain(+ '· 1 other workflow changed (not measured)</summary>',+ );+ expect(section).toContain(+ 'Other workflows: perf-nightly.yml added: runs on schedule (17 9 \\* \\* \\*); standard runners; its minutes are not measured here',+ );+ });++ it('renders job and workflow names from the pull request as literal text', () => {+ const section = renderCiImpact({+ ...impact,+ changes: [+ {+ name: '<img src=x>',+ change: 'added',+ before: null,+ after: runnerClass('ubuntu-latest'),+ },+ ],+ otherWorkflows: [+ describeWorkflow('[evil](https://x.example).yml', null, {+ on: 'push',+ jobs: {},+ }),+ ],+ });+ expect(section).toContain('<img src=x> added: standard');+ expect(section).toContain('\\[evil\\](https://x.example).yml added');+ expect(section).not.toContain('<img');+ });++ it('says so when a CI change could not be measured', () => {+ expect(renderCiImpact({unavailable: true})).toBe(+ '**CI impact:** not measured on this run; the run history or the workflow files could not be read.\n\n',+ );+ });++ it('names composite action and CI script changes as not measured', () => {+ const section = renderCiImpact({+ ...impact,+ changes: [],+ unmeasuredFiles: ['.github/actions/setup/action.yml'],+ });+ expect(section).toContain(+ '· 1 CI action or script file changed (not measured)</summary>',+ );+ expect(section).toContain(+ 'Not measured (the jobs that use these keep their recent minutes above): .github/actions/setup/action.yml.',+ );+ });++ it('renders nothing for a pull request that does not change CI', () => {+ expect(renderCiImpact(null)).toBe('');+ });+});diff --git a/.github/scripts/lib/main-artifact.cjs b/.github/scripts/lib/main-artifact.cjsindex 4bbd3b294a..5df9f3eb2e 100644--- a/.github/scripts/lib/main-artifact.cjs+++ b/.github/scripts/lib/main-artifact.cjs@@ -3,12 +3,12 @@/*** @file main-artifact.cjs* @input An Octokit client, the repository, an artifact name, and the commit- * it must belong to+ * it must belong to (optional: any commit on `main`)* @output The id of the workflow run on `main` that uploaded that artifact for- * that exact commit, or null+ * that exact commit (or any commit, without one), or null* @position Shared by the CI jobs that read what Main keeps for a merge base- * (its package builds, its package-size measurement), so each trusts the- * same thing: only Main's own run on `main` for this exact commit, in this+ * (its size records) and the PR report's daily CI history, so each trusts+ * the same thing: only Main's own run on `main` for this exact commit, in this* repository. A pull request's run can upload an artifact under any name.*/@@ -25,7 +25,7 @@ async function mainArtifactRunId({github, owner, repo, name, sha}) {candidate =>!candidate.expired &&candidate.workflow_run?.head_branch === 'main' &&- candidate.workflow_run?.head_sha === sha &&+ (sha == null || candidate.workflow_run?.head_sha === sha) &&candidate.workflow_run?.head_repository_id ===candidate.workflow_run?.repository_id,);diff --git a/.github/scripts/lib/main-artifact.test.mjs b/.github/scripts/lib/main-artifact.test.mjsindex 160d1accb7..4cad68ba16 100644--- a/.github/scripts/lib/main-artifact.test.mjs+++ b/.github/scripts/lib/main-artifact.test.mjs@@ -68,4 +68,23 @@ describe('mainArtifactRunId', () => {}),).resolves.toBeNull();});++ it("without a commit, takes Main's run at any commit, and still ignores a branch or a fork", async () => {+ const lookupAny = candidate =>+ mainArtifactRunId({+ github: github([candidate]),+ owner: 'meta',+ repo: 'astryx',+ name: 'x',+ });+ await expect(+ lookupAny(artifact({}, {head_sha: 'b'.repeat(40)})),+ ).resolves.toBe('42');+ await expect(+ lookupAny(artifact({}, {head_branch: 'feature'})),+ ).resolves.toBeNull();+ await expect(+ lookupAny(artifact({}, {head_repository_id: 2})),+ ).resolves.toBeNull();+ });});diff --git a/.github/scripts/lib/pr-preview-imports.test.mjs b/.github/scripts/lib/pr-preview-imports.test.mjsnew file mode 100644index 0000000000..54ebdd5212--- /dev/null+++ b/.github/scripts/lib/pr-preview-imports.test.mjs@@ -0,0 +1,63 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file pr-preview-imports.test.mjs+ * Contract: pr-preview.mjs, and every module it requires or statically imports,+ * loads only Node built-ins and relative files. The PR Comment workflow's resolve+ * and spec-only jobs import it without installing dependencies, so a+ * top-level package import would fail their report.+ */++import fs from 'node:fs';+import path from 'node:path';+import {describe, expect, it} from 'vitest';++const ENTRY = path.join(import.meta.dirname, 'pr-preview.mjs');++/**+ * Specifiers a module loads: static imports, and every require (on one line+ * or across lines, in any scope). Dynamic `import()` is lazy and left out.+ */+function topLevelSpecifiers(source) {+ const specs = [];+ for (const match of source.matchAll(+ /^import\s+(?:[\s\S]*?\s+from\s+)?['"]([^'"]+)['"]/gm,+ ))+ specs.push(match[1]);+ for (const match of source.matchAll(+ /\brequire\(\s*['"`]([^'"`]+)['"`]\s*\)/g,+ ))+ specs.push(match[1]);+ for (const match of source.matchAll(+ /createRequire\([^)]*\)\(\s*['"]([^'"]+)['"]\s*\)/g,+ ))+ specs.push(match[1]);+ return specs;+}++function resolveRelative(from, spec) {+ const base = path.resolve(path.dirname(from), spec);+ for (const candidate of [base, `${base}.js`, `${base}.cjs`, `${base}.mjs`])+ if (fs.existsSync(candidate) && fs.statSync(candidate).isFile())+ return candidate;+ throw new Error(`${spec} from ${from} does not resolve`);+}++describe('pr-preview.mjs top-level imports', () => {+ it('loads only Node built-ins and relative files, transitively', () => {+ const seen = new Set();+ const packages = [];+ const visit = file => {+ if (seen.has(file)) return;+ seen.add(file);+ for (const spec of topLevelSpecifiers(fs.readFileSync(file, 'utf8'))) {+ if (spec.startsWith('node:')) continue;+ if (spec.startsWith('.')) visit(resolveRelative(file, spec));+ else packages.push(`${path.basename(file)} → ${spec}`);+ }+ };+ visit(ENTRY);+ expect(seen.size).toBeGreaterThan(3);+ expect(packages).toEqual([]);+ });+});diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjsindex 6a2ff7181e..dd6acdd3c1 100644--- a/.github/scripts/lib/pr-preview.mjs+++ b/.github/scripts/lib/pr-preview.mjs@@ -8,6 +8,9 @@ import process from 'node:process';import {createRequire} from 'node:module';import {fileURLToPath} from 'node:url';+const require = createRequire(import.meta.url);+const ciImpact = require('./ci-impact.js');+const {previewLine} = createRequire(import.meta.url)('./pr-status.js');export const PR_ANALYSIS_MARKER = '<!-- astryx-pr-analysis -->';@@ -113,6 +116,206 @@ export async function headPreview({: {state: 'unknown', sha};}+/** Pull-request CI runs sampled for the CI-impact baseline. */+const CI_HISTORY_RUNS = 20;++/**+ * Recent pull-request CI runs: the latest CI_HISTORY_RUNS successful runs with+ * their jobs' timings, and how many runs did work (succeeded or failed) in the+ * seven whole days before today. Read once a day: the+ * comment job keeps it as an artifact and passes it back in `cached`.+ */+export async function ciHistory({+ github,+ owner,+ repo,+ cached,+ now = new Date(),+}) {+ if (cached?.runs) return cached;+ // The seven whole days before today (UTC), so runs a day divides seven+ // days of runs by seven.+ const day = offset =>+ new Date(now.getTime() - offset * 24 * 3600 * 1000)+ .toISOString()+ .slice(0, 10);+ const window = `${day(7)}..${day(1)}`;+ // Runs that did work: a draft's or an out-of-scope run concludes skipped.+ const count = async status =>+ (+ await github.rest.actions.listWorkflowRuns({+ owner,+ repo,+ workflow_id: 'ci.yml',+ event: 'pull_request',+ status,+ created: window,+ per_page: 1,+ })+ ).data.total_count;+ const runsLast7Days = (await count('success')) + (await count('failure'));+ const {data: recent} = await github.rest.actions.listWorkflowRuns({+ owner,+ repo,+ workflow_id: 'ci.yml',+ event: 'pull_request',+ status: 'success',+ per_page: CI_HISTORY_RUNS,+ });+ const runs = [];+ for (const run of recent.workflow_runs.slice(0, CI_HISTORY_RUNS)) {+ const jobs = await sourceRunJobs({github, owner, repo, runId: run.id});+ if (jobs)+ runs.push({+ run_started_at: run.run_started_at,+ updated_at: run.updated_at,+ jobs,+ });+ }+ return {runs, runsLast7Days};+}++async function workflowAt({+ github,+ owner,+ repo,+ ref,+ file = '.github/workflows/ci.yml',+}) {+ try {+ const {data} = await github.rest.repos.getContent({+ owner,+ repo,+ path: file,+ ref,+ });+ // Loaded here, not at the top: the PR Comment workflow's resolve and+ // spec-only jobs import this module without installing dependencies.+ const {default: yaml} = await import('yaml');+ return yaml.parse(Buffer.from(data.content, 'base64').toString('utf8'));+ } catch (error) {+ // A workflow the pull request adds has no base, one it removes no head.+ if (error?.status === 404 || /Not Found|404/.test(String(error?.message)))+ return null;+ throw error;+ }+}++/**+ * The CI impact of a pull request that changes CI, or null for any other.+ * Reads the changed files, CI's workflow at the merge base and the head, the+ * source run, and the history (computed here only when `historyPath` holds+ * none, then written there for the comment job to keep).+ */+export async function gatherCiImpact({+ github,+ owner,+ repo,+ identity,+ jobs,+ historyPath,+}) {+ // The file list decides whether this is a CI change at all (one read for+ // every pull request); unreadable, the section is left out.+ let names;+ try {+ const files = await github.paginate(github.rest.pulls.listFiles, {+ owner,+ repo,+ pull_number: identity.prNumber,+ per_page: 100,+ });+ names = files.map(file => file.filename);+ } catch {+ return null;+ }+ if (!ciImpact.isCiChange(names)) return null;+ try {+ const cached =+ historyPath && fs.existsSync(historyPath) ? readJSON(historyPath) : null;+ const history = await ciHistory({github, owner, repo, cached});+ if (historyPath && !cached) {+ fs.mkdirSync(path.dirname(historyPath), {recursive: true});+ fs.writeFileSync(historyPath, JSON.stringify(history));+ }+ // The before side is the merge base, the tree the head was cut from.+ const {data: comparison} =+ await github.rest.repos.compareCommitsWithBasehead({+ owner,+ repo,+ basehead: `${identity.baseSha}...${identity.headSha}`,+ });+ const mergeBase = comparison.merge_base_commit.sha;+ const [base, head] = await Promise.all([+ workflowAt({github, owner, repo, ref: mergeBase}),+ workflowAt({github, owner, repo, ref: identity.headSha}),+ ]);+ const {data: run} = await github.rest.actions.getWorkflowRun({+ owner,+ repo,+ run_id: identity.sourceRunId,+ });+ // Every other changed workflow: when it runs and on which runners; its+ // minutes are not measured (the history covers CI's own workflow).+ const otherWorkflows = [];+ for (const file of names.filter(+ name =>+ /^\.github\/workflows\/[^/]+\.ya?ml$/.test(name) &&+ name !== '.github/workflows/ci.yml',+ )) {+ const [before, after] = await Promise.all([+ workflowAt({github, owner, repo, ref: mergeBase, file}),+ workflowAt({github, owner, repo, ref: identity.headSha, file}),+ ]);+ otherWorkflows.push(+ ciImpact.describeWorkflow(+ file.replace('.github/workflows/', ''),+ before,+ after,+ ),+ );+ }+ const started = Date.parse(run.run_started_at ?? '');+ const finished = Date.parse(run.updated_at ?? '');+ const impact = ciImpact.estimateImpact({+ diff: ciImpact.diffWorkflow(base, head),+ history: ciImpact.summarizeHistory(history),+ currentJobs: jobs,+ // Time to green compares like with like: a run that failed or was+ // cancelled has no time to green.+ currentWallMinutes:+ run.conclusion === 'success' &&+ Number.isFinite(started) &&+ Number.isFinite(finished)+ ? (finished - started) / 60000+ : null,+ });+ return {+ ...impact,+ otherWorkflows,+ // Composite actions and CI scripts change what jobs do, but this run's+ // minutes are not compared with theirs: named, not measured.+ unmeasuredFiles: names.filter(+ name =>+ ciImpact.CI_PATH.test(name) &&+ !/^\.github\/workflows\/[^/]+\.ya?ml$/.test(name),+ ),+ flags: [+ ...impact.flags,+ ...otherWorkflows+ .filter(workflow => workflow.nonStandard)+ .map(+ workflow =>+ `${ciImpact.inlineName(workflow.file)} moves a job onto a non-standard runner`,+ ),+ ],+ };+ } catch {+ // A CI change whose impact could not be read says so.+ return {unavailable: true};+ }+}+/*** The frames publish-frames.mjs published, written for the generator, only* when their URL is this pull request's assets branch for this exact head and@@ -163,6 +366,8 @@ export async function sourceRunJobs({github, owner, repo, runId}) {name: job.name,status: job.status,conclusion: job.conclusion,+ started_at: job.started_at,+ completed_at: job.completed_at,steps: (job.steps || []).map(step => ({name: step.name,conclusion: step.conclusion,@@ -617,6 +822,7 @@ export async function reconcilePrComment({visualFrames = null,a11yPath = 'a11y/a11y-report.json',visualPath = 'trusted-visual/verdict.json',+ ciHistoryPath = 'ci-history/ci-history.json',createIfMissing = true,fallbackMessage,touchedPath = 'touched-components/touched-components.json',@@ -680,6 +886,16 @@ export async function reconcilePrComment({repo,runId: identity.sourceRunId,});+ const impact = await gatherCiImpact({+ github,+ owner,+ repo,+ identity,+ jobs,+ historyPath: ciHistoryPath,+ });+ const impactPath = path.join(inputs, 'ci-impact.json');+ if (impact) fs.writeFileSync(impactPath, JSON.stringify(impact));// Frames only from this PR's assets branch for this exact head.const framesPath = trustedFrames({frames: visualFrames, identity, inputs});const previewPath = path.join(inputs, 'preview.json');@@ -725,6 +941,7 @@ export async function reconcilePrComment({: []),...(preview ? ['--preview', previewPath] : []),...(jobs ? ['--jobs', jobsPath] : []),+ ...(impact ? ['--ci-impact', impactPath] : []),'--source-conclusion',identity.sourceConclusion,'--run-url',diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjsindex 510e5d918e..b675fb98be 100644--- a/.github/scripts/lib/pr-preview.test.mjs+++ b/.github/scripts/lib/pr-preview.test.mjs@@ -11,6 +11,7 @@ import yaml from 'yaml';import {branchPreviewOrigin,+ ciHistory,confirmOrStandDown,confirmSourceRunIdentity,PR_ANALYSIS_MARKER,@@ -205,6 +206,8 @@ async function reconcile({packageSize,touched,apiChanges,+ extendGithub,+ ciHistoryPath,} = {}) {const value = previewAvailable? {...requested, headRepository: requested.baseRepository}@@ -236,6 +239,7 @@ async function reconcile({value: githubFixtureValue,comments,});+ extendGithub?.(github);const core = {info: vi.fn(), warning: vi.fn()};const result = await reconcilePrComment({github,@@ -256,6 +260,9 @@ async function reconcile({createIfMissing,fallbackMessage,execute,+ ciHistoryPath:+ ciHistoryPath ??+ path.join(path.dirname(paths.analysis), 'ci-history.json'),});return {result, state, core, github, paths};}@@ -577,7 +584,12 @@ describe('PR Comment keeps its outcome off main', () => {'comment','report',];- const OPTIONAL = new Set(['actions/download-artifact@v8']);+ // Optional: artifact downloads a run may not have, and keeping the CI+ // history, which is a cache.+ const OPTIONAL = new Set([+ 'actions/download-artifact@v8',+ 'actions/upload-artifact@v7',+ ]);it.each(LANE)('never fails the run from the %s job', name => {for (const step of PR_COMMENT_WORKFLOW.jobs[name].steps) {@@ -1483,3 +1495,137 @@ describe('visual frames', () => {expect(jobs.comment.needs).toEqual(['resolve', 'frames']);});});++describe('CI impact in the report', () => {+ const at = minute =>+ new Date(Date.UTC(2026, 9, 31, 12, minute)).toISOString();+ const workflow = text => ({content: Buffer.from(text).toString('base64')});+ const BASE_CI =+ 'on: pull_request\njobs:\n lint:\n runs-on: ubuntu-latest\n steps: [{run: a}]\n';+ const HEAD_CI = `${BASE_CI} extra:\n runs-on: 4-core-ubuntu\n steps: [{run: b}]\n`;+ const ciChange = github => {+ github.rest.pulls.listFiles = vi.fn(async () => ({+ data: [{filename: '.github/workflows/ci.yml'}],+ }));+ github.rest.repos = {+ compareCommitsWithBasehead: vi.fn(async () => ({+ data: {merge_base_commit: {sha: 'c'.repeat(40)}},+ })),+ getContent: vi.fn(async ({ref}) => ({+ data: workflow(ref === 'c'.repeat(40) ? BASE_CI : HEAD_CI),+ })),+ };+ github.rest.actions.listWorkflowRuns = vi.fn(async ({per_page}) => ({+ data: {+ total_count: 70,+ workflow_runs:+ per_page === 1+ ? []+ : [{id: 1, run_started_at: at(0), updated_at: at(10)}],+ },+ }));+ github.rest.actions.listJobsForWorkflowRun = vi.fn(async () => ({+ data: [+ {+ name: 'lint',+ conclusion: 'success',+ started_at: at(0),+ completed_at: at(2),+ },+ {+ name: 'extra',+ conclusion: 'success',+ started_at: at(0),+ completed_at: at(3),+ },+ ],+ }));+ };++ it('reaches the comment for a pull request that changes CI, with the merge base as before', async () => {+ let seen;+ const {github} = await reconcile({+ extendGithub: ciChange,+ execute: (_node, args) => {+ seen = args;+ const file = args[args.indexOf('--ci-impact') + 1];+ return `## PR Analysis Report\n${fs.readFileSync(file, 'utf8')}`;+ },+ });+ expect(seen).toContain('--ci-impact');+ const impact = JSON.parse(+ fs.readFileSync(seen[seen.indexOf('--ci-impact') + 1], 'utf8'),+ );+ expect(impact.flags).toContain('extra moves onto a non-standard runner');+ expect(github.rest.repos.getContent).toHaveBeenCalledWith(+ expect.objectContaining({ref: 'c'.repeat(40)}),+ );+ });++ it('names a composite-action-only change as not measured', async () => {+ let seen;+ await reconcile({+ extendGithub: github => {+ ciChange(github);+ github.rest.pulls.listFiles = vi.fn(async () => ({+ data: [{filename: '.github/actions/setup/action.yml'}],+ }));+ },+ execute: (_node, args) => {+ seen = args;+ return '## PR Analysis Report\n';+ },+ });+ const impact = JSON.parse(+ fs.readFileSync(seen[seen.indexOf('--ci-impact') + 1], 'utf8'),+ );+ expect(impact.unmeasuredFiles).toEqual([+ '.github/actions/setup/action.yml',+ ]);+ });++ it('passes no impact for a pull request that does not change CI', async () => {+ let seen;+ await reconcile({+ extendGithub: github => {+ ciChange(github);+ github.rest.pulls.listFiles = vi.fn(async () => ({+ data: [{filename: 'packages/core/src/Card/Card.tsx'}],+ }));+ },+ execute: (_node, args) => {+ seen = args;+ return '## PR Analysis Report\n';+ },+ });+ expect(seen).not.toContain('--ci-impact');+ });+});++describe('CI history window', () => {+ it('counts the seven whole days before today', async () => {+ const calls = [];+ const github = {+ rest: {+ actions: {+ listWorkflowRuns: vi.fn(async request => {+ calls.push(request);+ return {data: {total_count: 7, workflow_runs: []}};+ }),+ },+ },+ };+ const history = await ciHistory({+ github,+ owner: 'meta',+ repo: 'astryx',+ now: new Date('2026-11-01T02:00:00Z'),+ });+ const counted = calls.filter(call => call.per_page === 1);+ expect(counted.map(call => call.created)).toEqual([+ '2026-10-25..2026-10-31',+ '2026-10-25..2026-10-31',+ ]);+ expect(history.runsLast7Days).toBe(14);+ });+});diff --git a/.github/scripts/lib/runner-labels.cjs b/.github/scripts/lib/runner-labels.cjsnew file mode 100644index 0000000000..077f8b2e39--- /dev/null+++ b/.github/scripts/lib/runner-labels.cjs@@ -0,0 +1,73 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file runner-labels.cjs+ * @input A workflow job (its `runs-on` and matrix)+ * @output Every runner label the job can resolve to, and the set of GitHub's+ * standard hosted runner labels+ * @position Shared by ci-free-runners.test.mjs (every job runs on a standard+ * runner unless allowed) and lib/ci-impact.js (the PR report's runner+ * classes), so both read a job's runners the same way.+ */++/** GitHub's standard hosted runner labels. */+const STANDARD_RUNNERS = new Set([+ 'ubuntu-latest',+ 'ubuntu-slim',+ 'ubuntu-24.04',+ 'ubuntu-22.04',+ 'ubuntu-24.04-arm',+ 'ubuntu-22.04-arm',+ 'macos-latest',+ 'macos-15',+ 'macos-14',+ 'windows-latest',+]);++/**+ * Every runner label a job can resolve to. Matrix values include `include`+ * entries; an expression of the shape `cond && 'a' || 'b'` yields both+ * labels. Anything that does not resolve (`${{ vars.X }}`, `${{ inputs.X }}`,+ * `${{ vars.X || 'ubuntu-latest' }}`, a computed matrix) is returned raw, so+ * a caller can tell it is unknown rather than read it as standard.+ */+function labelsOf(job) {+ const runsOn = job?.['runs-on'];+ if (runsOn == null) return [];+ const labels = [];+ for (const value of [runsOn].flat().map(String)) {+ const matrix = /^\$\{\{\s*matrix\.([a-zA-Z0-9_-]+)\s*\}\}$/.exec(value);+ if (matrix) {+ const key = matrix[1];+ const m = job.strategy?.matrix;+ const found =+ m && typeof m === 'object'+ ? [+ ...[m[key] ?? []].flat(),+ ...[m.include ?? []].flat().map(entry => entry?.[key]),+ // A computed include can add any runner: fail closed.+ ...(typeof m.include === 'string' ? [value] : []),+ ]+ .filter(item => item != null)+ .map(String)+ : [];+ labels.push(...(found.length > 0 ? found : [value]));+ } else if (value.includes('${{')) {+ const pair =+ /^\$\{\{\s*([^|]+?)\s*&&\s*'([^']+)'\s*\|\|\s*'([^']+)'\s*\}\}$/.exec(+ value,+ );+ labels.push(...(pair ? [pair[2], pair[3]] : [value]));+ } else {+ labels.push(value);+ }+ }+ return labels;+}++/** Whether a label is an expression `labelsOf` could not resolve. */+function isUnresolved(label) {+ return String(label).includes('${{');+}++module.exports = {STANDARD_RUNNERS, labelsOf, isUnresolved};diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.ymlindex 0edd6b8e3f..3eed1a9a35 100644--- a/.github/workflows/pr-comment.yml+++ b/.github/workflows/pr-comment.yml@@ -306,6 +306,7 @@ jobs:steps.identity.outcome == 'failure' ||steps.cross-check.outcome == 'failure' ||steps.visual.outcome == 'failure' ||+ steps.ci-history.outcome == 'failure' ||steps.publish.outcome == 'failure' }}steps:- name: Checkout trusted default-branch code@@ -478,6 +479,37 @@ jobs:--run-id "$RUN_ID" \--run-attempt "$RUN_ATTEMPT"+ # The CI-impact baseline (recent pull-request runs) is read once a day+ # and kept as an artifact, so later CI-changing PRs that day reuse it.+ - name: Find today's CI history+ id: ci-history+ continue-on-error: true+ if: steps.identity.outputs.valid == 'true'+ uses: actions/github-script@v9+ with:+ retries: 3+ script: |+ // Wait out the repository's shared API rate limit (lib/rate-limit-backoff.cjs).+ require(require('node:path').join(process.env.GITHUB_WORKSPACE, '.github/scripts/lib/rate-limit-backoff.cjs'))+ .installRateLimitBackoff({github, core});+ // Only this repository's own runs on main keep the history, at any+ // commit (lib/main-artifact.cjs).+ const {mainArtifactRunId} = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/lib/main-artifact.cjs`);+ const name = `ci-history-${new Date().toISOString().slice(0, 10)}`;+ const runId = await mainArtifactRunId({github, ...context.repo, name});+ core.setOutput('name', name);+ core.setOutput('run_id', runId ?? '');++ - name: Download today's CI history+ if: steps.identity.outputs.valid == 'true' && steps.ci-history.outputs.run_id != ''+ continue-on-error: true+ uses: actions/download-artifact@v8+ with:+ name: ${{ steps.ci-history.outputs.name }}+ run-id: ${{ steps.ci-history.outputs.run_id }}+ github-token: ${{ secrets.GITHUB_TOKEN }}+ path: ci-history+- name: Generate and post PR commentid: publishcontinue-on-error: true@@ -528,6 +560,15 @@ jobs:},});+ - name: Keep today's CI history+ if: steps.identity.outputs.valid == 'true' && steps.ci-history.outputs.name != '' && steps.ci-history.outputs.run_id == '' && hashFiles('ci-history/ci-history.json') != ''+ continue-on-error: true+ uses: actions/upload-artifact@v7+ with:+ name: ${{ steps.ci-history.outputs.name }}+ path: ci-history/ci-history.json+ retention-days: 2+# The CI-completion lane runs on workflow_run, so GitHub attaches its jobs to# main's latest commit. Main's checks must show main's health, not a pull# request's. The jobs above therefore never fail the run: every fallible stepdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 05c65e6fdb..48952ff2d8 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -932,6 +932,15 @@ Storybook story for anything you added or altered.takes effect on the next push or a close and reopen, and removing it stopsthat suite from the next run. When the accessibility audit did not run, thepull-request comment names `ci:run-a11y`.+- **A pull request that changes CI states its CI impact.** When it changes a+ workflow, a composite action or a CI script, its description carries a+ `CI impact:` line: each changed job's runner class (standard or+ non-standard), job-minutes per update and runner-pool usage before and+ after, and time to green. The pull-request comment's CI impact section+ measures these for `ci.yml` from recent runs, names other changed workflows,+ composite actions and CI scripts as not measured, and flags a job moving onto+ a non-standard runner, more than 10% added runner-pool usage, or more than two minutes+ added to time to green.- **Pull requests from a branch in this repository run CI without the label**,since pushing a branch here needs write access. Maintainers who work from afork either push their branches here or apply `ci:run` themselves.diff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.mdindex fa8497bd40..284045ebc9 100644--- a/docs/specs/AST-030/spec.md+++ b/docs/specs/AST-030/spec.md@@ -321,6 +321,22 @@ they are separately named.required check would count as passing. Events other than a pull request (apush to `main`, a merge-group entry, a dispatch) are unaffected, and a fork'scode never runs with the repository's secrets.+- **FR19 — A pull request that changes CI states its CI impact.** A pull+ request that changes a workflow, a composite action or a CI script carries+ its CI impact in its description's `CI impact:` line: each changed job's+ runner class (standard GitHub-hosted or non-standard), job-minutes per+ pull-request update before and after, the change in runner-pool usage+ (runner-minutes a day and average concurrent jobs) and time to green. The+ pull-request comment measures these for CI's own workflow (`ci.yml`) from+ its change and recent pull-request runs, without starting a job: a changed job counts as often as+ it ran in those runs, time to green compares with runs that ran the same+ jobs, and another changed workflow, a composite action or a CI script is+ named as not measured.+ A `node-tooling` pull request (FR8) gets no pull-request comment, so its+ description's line stands alone. The comment flags a job moving onto a+ non-standard runner, more than 10% added runner-pool usage, or more than two+ minutes added to time to green. A flag asks the reviewer to weigh the cost;+ it fails no check.### Platform support@@ -372,6 +388,7 @@ read change (FR14).| FR15 | workflow contract and reconciler tests | ENGOWNER spec PR; owner command; other comment; fork review; non-spec PR | `review-required` succeeds while spec approval is missing, another workflow publishes a review context, a fork review starts the spec checks, or a non-command comment starts a runner || FR16 | `heavy-caps-action.test.mjs` cap cases and heavy-job contract | sixteenth-oldest PR; 101st heavy run of the day, beside cancelled, light and other-branch runs; agent lane and its five-PR and forty-run backstops; an engineering owner, including the shared login past those backstops, a commented-out or trailing-comment handle and an unreadable roster; bot; the deciding run; re-run; failed lookup | a heavy suite that runs ahead of the author's fifteen oldest PRs, or past the daily cap, a capped job that passes, an engineering owner or a re-run that is held, or a roster read from the pull request's head fails || FR17 | `ci-pr-gate.test.mjs` over every `pull_request` workflow and the shared conditions | branch here, fork with and without `ci:run`, another label, draft with the label, ready, push, merge group, dispatch | a job that can start on a pull request without the condition, a label trigger, a required job that skips for a pull request that may not run CI, or a draft or unlabelled fork that runs fails |+| FR19 | `ci-impact.test.mjs`, `pr-preview.test.mjs` and the maintenance template | CI change and non-CI change; standard, non-standard and expression runners; added, removed and changed jobs; trigger change | a CI change reports no impact, a move onto a non-standard runner or a pool or time-to-green rise past its threshold goes unflagged, or the section names a price |## Decision log