Skip to content

An open source design system that's fully customizable and agent ready

TypeScript

ci: centralize gh-pages publishing (#5629)

Cindy Zhang677d68ea8f64
diff --git a/.github/scripts/gh-pages-publisher.mjs b/.github/scripts/gh-pages-publisher.mjs
new file mode 100755
index 0000000000..86cc44afd4
--- /dev/null
+++ b/.github/scripts/gh-pages-publisher.mjs
@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {main} from './lib/gh-pages-publisher.mjs';
+
+await main();
diff --git a/.github/scripts/lib/gh-pages-publisher.mjs b/.github/scripts/lib/gh-pages-publisher.mjs
new file mode 100755
index 0000000000..9aa700b322
--- /dev/null
+++ b/.github/scripts/lib/gh-pages-publisher.mjs
@@ -0,0 +1,789 @@
+#!/usr/bin/env node
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+const PAGES_BRANCH = 'gh-pages';
+const QUEUE_REL = path.join('.astryx-gh-pages', 'publication-queue');
+const HOLDER_NAME = 'holder.json';
+const RUN_ID = /^[1-9][0-9]*$/;
+const SCOPE = /^[a-z0-9][a-z0-9._/-]*$/;
+const BOT_NAME = 'github-actions[bot]';
+const BOT_EMAIL = 'github-actions[bot]@users.noreply.github.com';
+const DEFAULT_WAIT_MS = 75 * 60 * 1000;
+
+function refuse(message) {
+ throw new Error(`gh-pages publisher refused: ${message}`);
+}
+
+function run(command, args, options = {}) {
+ const result = spawnSync(command, args, {
+ encoding: 'utf8',
+ ...options,
+ });
+ if (result.status !== 0) {
+ refuse(
+ `${command} ${args[0] ?? ''} failed: ${
+ result.stderr?.trim() ||
+ result.stdout?.trim() ||
+ `exit ${result.status}`
+ }`,
+ );
+ }
+ return result.stdout.trim();
+}
+
+function tryRun(command, args, options = {}) {
+ return spawnSync(command, args, {encoding: 'utf8', ...options});
+}
+
+function git(cwd, ...args) {
+ return run('git', ['-C', cwd, ...args]);
+}
+
+function configureGitIdentity(cwd) {
+ git(cwd, 'config', 'user.name', BOT_NAME);
+ git(cwd, 'config', 'user.email', BOT_EMAIL);
+}
+
+function sleep(ms) {
+ return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function validateIdentity(repository, runId, scope) {
+ if (typeof repository !== 'string' || !/^[^/]+\/[^/]+$/.test(repository)) {
+ refuse('repository identity is invalid');
+ }
+ if (!Number.isSafeInteger(runId) || runId <= 0) {
+ refuse('current run id is invalid');
+ }
+ if (typeof scope !== 'string' || !SCOPE.test(scope) || scope.includes('..')) {
+ refuse('publication scope is invalid');
+ }
+}
+
+function ticketValue(repository, runId, scope) {
+ return {version: 1, repository, runId, scope};
+}
+
+function validTicket(value, repository, runId, scope = value?.scope) {
+ return (
+ Number.isSafeInteger(runId) &&
+ runId > 0 &&
+ value?.version === 1 &&
+ value.repository === repository &&
+ value.runId === runId &&
+ value.scope === scope &&
+ typeof scope === 'string' &&
+ SCOPE.test(scope) &&
+ !scope.includes('..')
+ );
+}
+
+function readJSON(file) {
+ try {
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
+ } catch {
+ return null;
+ }
+}
+
+export function orderedTickets(entries, repository) {
+ if (!Array.isArray(entries)) refuse('queue entries are invalid');
+ const tickets = [];
+ const seen = new Set();
+ for (const entry of entries) {
+ if (!RUN_ID.test(entry?.name ?? '')) continue;
+ const runId = Number(entry.name);
+ if (!validTicket(entry.value, repository, runId)) {
+ refuse(`queue ticket ${entry.name} has invalid identity`);
+ }
+ if (seen.has(runId)) refuse('queue repeats a run id');
+ seen.add(runId);
+ tickets.push(entry.value);
+ }
+ return tickets.sort((a, b) => a.runId - b.runId);
+}
+
+export function publicationBlockers(entries, repository, currentRunId, scope) {
+ validateIdentity(repository, currentRunId, scope);
+ const tickets = orderedTickets(entries, repository);
+ if (
+ !tickets.some(
+ ticket => ticket.runId === currentRunId && ticket.scope === scope,
+ )
+ ) {
+ refuse(`queue ticket for run ${currentRunId} is missing`);
+ }
+ return tickets.filter(ticket => ticket.runId < currentRunId);
+}
+
+export function isTerminalRun(status) {
+ return status === 'completed' || status === 'missing';
+}
+
+function queueState(checkout, repository) {
+ const root = path.join(checkout, QUEUE_REL);
+ if (!fs.existsSync(root)) return {entries: [], invalid: [], holder: null};
+ const entries = [];
+ const invalid = [];
+ let holder = null;
+ for (const entry of fs.readdirSync(root, {withFileTypes: true})) {
+ if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
+ const value = readJSON(path.join(root, entry.name));
+ if (entry.name === HOLDER_NAME) {
+ if (!validTicket(value, repository, value?.runId)) {
+ refuse('publication holder has invalid identity');
+ }
+ holder = value;
+ continue;
+ }
+ const name = entry.name.slice(0, -'.json'.length);
+ if (!RUN_ID.test(name)) {
+ invalid.push(entry.name);
+ continue;
+ }
+ const runId = Number(name);
+ if (!validTicket(value, repository, runId)) {
+ invalid.push(entry.name);
+ continue;
+ }
+ entries.push({name, value});
+ }
+ return {entries, invalid, holder};
+}
+
+function remoteURL({repository, token, remoteURL}) {
+ return (
+ remoteURL ??
+ process.env.ASTRYX_GH_PAGES_REMOTE_URL ??
+ `https://x-access-token:${token}@github.com/${repository}.git`
+ );
+}
+
+function checkoutPages({
+ repository,
+ token,
+ tempRoot,
+ sparsePaths = [],
+ noCheckout = false,
+ prefix = 'gh-pages-',
+ remoteURL: configuredRemoteURL,
+}) {
+ const checkout = fs.mkdtempSync(path.join(tempRoot, prefix));
+ const args = [
+ 'clone',
+ '--quiet',
+ '--depth=1',
+ '--filter=blob:none',
+ '--single-branch',
+ '--branch',
+ PAGES_BRANCH,
+ ];
+ if (noCheckout) {
+ args.push('--no-checkout');
+ } else if (sparsePaths.length > 0) {
+ args.push('--sparse');
+ }
+ args.push(
+ remoteURL({repository, token, remoteURL: configuredRemoteURL}),
+ checkout,
+ );
+ run('git', args);
+ if (noCheckout) {
+ git(checkout, 'sparse-checkout', 'init', '--cone');
+ git(checkout, 'sparse-checkout', 'set', ...sparsePaths);
+ git(checkout, 'checkout', PAGES_BRANCH);
+ } else if (sparsePaths.length > 0) {
+ git(checkout, 'sparse-checkout', 'set', ...sparsePaths);
+ }
+ return checkout;
+}
+
+async function mutateQueue({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ mutate,
+}) {
+ validateIdentity(repository, runId, scope);
+ for (let attempt = 1; attempt <= 10; attempt += 1) {
+ const checkout = checkoutPages({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ sparsePaths: [QUEUE_REL],
+ prefix: 'gh-pages-queue-',
+ });
+ try {
+ const mutation = mutate(checkout);
+ if (!mutation.changed) return mutation.value;
+ configureGitIdentity(checkout);
+ git(checkout, 'add', '-A', QUEUE_REL);
+ git(checkout, 'commit', '-qm', 'ci: update gh-pages publication queue');
+ const pushed = tryRun('git', [
+ '-C',
+ checkout,
+ 'push',
+ 'origin',
+ PAGES_BRANCH,
+ ]);
+ if (pushed.status === 0) return mutation.value;
+ if (permissionDenied(pushed)) {
+ refuse('push to gh-pages denied while updating the publication queue');
+ }
+ } finally {
+ fs.rmSync(checkout, {recursive: true, force: true});
+ }
+ await sleep(attempt * 1000);
+ }
+ refuse('could not update the publication queue after 10 attempts');
+}
+
+export async function enqueuePublication(options) {
+ const {repository, runId, scope} = options;
+ await mutateQueue({
+ ...options,
+ mutate(checkout) {
+ const file = path.join(checkout, QUEUE_REL, `${runId}.json`);
+ if (fs.existsSync(file)) {
+ if (!validTicket(readJSON(file), repository, runId, scope)) {
+ refuse('existing queue ticket has invalid identity');
+ }
+ return {changed: false};
+ }
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(
+ file,
+ `${JSON.stringify(ticketValue(repository, runId, scope), null, 2)}\n`,
+ );
+ return {changed: true};
+ },
+ });
+ process.stdout.write(
+ `Queued gh-pages publication run ${runId} (${scope}).\n`,
+ );
+}
+
+export async function releasePublication(options) {
+ const {repository, runId, scope} = options;
+ await mutateQueue({
+ ...options,
+ mutate(checkout) {
+ const root = path.join(checkout, QUEUE_REL);
+ const ticket = path.join(root, `${runId}.json`);
+ const holder = path.join(root, HOLDER_NAME);
+ let changed = false;
+ if (fs.existsSync(ticket)) {
+ fs.rmSync(ticket);
+ changed = true;
+ }
+ const held = readJSON(holder);
+ if (held?.runId === runId && held?.scope === scope) {
+ fs.rmSync(holder);
+ changed = true;
+ }
+ return {changed};
+ },
+ });
+ process.stdout.write(
+ `Released gh-pages publication run ${runId} (${scope}).\n`,
+ );
+}
+
+async function removeInvalid({
+ repository,
+ runId,
+ scope,
+ names,
+ token,
+ tempRoot,
+ remoteURL,
+}) {
+ await mutateQueue({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ mutate(checkout) {
+ let changed = false;
+ for (const name of names) {
+ const file = path.join(checkout, QUEUE_REL, name);
+ if (fs.existsSync(file)) {
+ fs.rmSync(file);
+ changed = true;
+ }
+ }
+ return {changed};
+ },
+ });
+}
+
+async function claimPublication(options) {
+ const {repository, runId, scope} = options;
+ return mutateQueue({
+ ...options,
+ mutate(checkout) {
+ const state = queueState(checkout, repository);
+ if (state.invalid.length > 0) return {changed: false, value: false};
+ if (state.holder) {
+ return {
+ changed: false,
+ value: state.holder.runId === runId && state.holder.scope === scope,
+ };
+ }
+ const tickets = orderedTickets(state.entries, repository);
+ if (tickets[0]?.runId !== runId || tickets[0]?.scope !== scope) {
+ return {changed: false, value: false};
+ }
+ fs.writeFileSync(
+ path.join(checkout, QUEUE_REL, HOLDER_NAME),
+ `${JSON.stringify(ticketValue(repository, runId, scope), null, 2)}\n`,
+ );
+ return {changed: true, value: true};
+ },
+ });
+}
+
+function runStatus(repository, runId) {
+ const result = tryRun('gh', [
+ 'api',
+ `repos/${repository}/actions/runs/${runId}`,
+ ]);
+ if (result.status === 0) {
+ try {
+ return JSON.parse(result.stdout).status;
+ } catch (error) {
+ refuse(`cannot parse queued run ${runId}: ${error.message}`);
+ }
+ }
+ if (/HTTP 404|Not Found/i.test(result.stderr ?? '')) return 'missing';
+ refuse(
+ `cannot resolve queued run ${runId}: ${
+ result.stderr?.trim() || result.stdout?.trim() || `exit ${result.status}`
+ }`,
+ );
+}
+
+export async function waitForPublicationTurn({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ timeoutMs = DEFAULT_WAIT_MS,
+}) {
+ validateIdentity(repository, runId, scope);
+ const started = Date.now();
+ const checkTimeout = blocker => {
+ if (Date.now() - started >= timeoutMs) {
+ refuse(
+ `timed out behind gh-pages publication run ${blocker ?? 'unknown'}`,
+ );
+ }
+ };
+ while (true) {
+ const checkout = checkoutPages({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ sparsePaths: [QUEUE_REL],
+ prefix: 'gh-pages-queue-',
+ });
+ let state;
+ try {
+ state = queueState(checkout, repository);
+ } finally {
+ fs.rmSync(checkout, {recursive: true, force: true});
+ }
+ if (state.invalid.length > 0) {
+ checkTimeout(state.holder?.runId);
+ await removeInvalid({
+ repository,
+ runId,
+ scope,
+ names: state.invalid,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ continue;
+ }
+ if (state.holder?.runId === runId && state.holder?.scope === scope) {
+ process.stdout.write(
+ `gh-pages publication turn acquired by run ${runId}.\n`,
+ );
+ return;
+ }
+ if (state.holder) {
+ if (isTerminalRun(runStatus(repository, state.holder.runId))) {
+ checkTimeout(state.holder.runId);
+ await releasePublication({
+ repository,
+ runId: state.holder.runId,
+ scope: state.holder.scope,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ continue;
+ }
+ } else {
+ const blockers = publicationBlockers(
+ state.entries,
+ repository,
+ runId,
+ scope,
+ );
+ let pruned = false;
+ for (const blocker of blockers) {
+ if (isTerminalRun(runStatus(repository, blocker.runId))) {
+ checkTimeout(blocker.runId);
+ await releasePublication({
+ repository,
+ runId: blocker.runId,
+ scope: blocker.scope,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ pruned = true;
+ }
+ }
+ if (pruned) continue;
+ if (
+ blockers.length === 0 &&
+ (await claimPublication({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ }))
+ ) {
+ process.stdout.write(
+ `gh-pages publication turn acquired by run ${runId}.\n`,
+ );
+ return;
+ }
+ }
+ const blocker =
+ state.holder?.runId ??
+ orderedTickets(state.entries, repository)[0]?.runId;
+ checkTimeout(blocker);
+ process.stdout.write(`Waiting for gh-pages publication run ${blocker}.\n`);
+ await sleep(30000);
+ }
+}
+
+function copyContents(source, destination) {
+ fs.mkdirSync(destination, {recursive: true});
+ for (const entry of fs.readdirSync(source, {withFileTypes: true})) {
+ fs.cpSync(
+ path.join(source, entry.name),
+ path.join(destination, entry.name),
+ {
+ recursive: true,
+ force: true,
+ },
+ );
+ }
+}
+
+function permissionDenied(result) {
+ const text = `${result.stderr ?? ''}\n${result.stdout ?? ''}`;
+ return /denied|403|permission|not authorized/i.test(text);
+}
+
+function pushOrRetry(result) {
+ if (result.status === 0) return true;
+ if (permissionDenied(result)) {
+ refuse('push to gh-pages denied');
+ }
+ return false;
+}
+
+function commitIfNeeded(checkout, message, addArgs) {
+ configureGitIdentity(checkout);
+ git(checkout, 'add', ...addArgs);
+ const diff = tryRun('git', ['-C', checkout, 'diff', '--cached', '--quiet']);
+ if (diff.status === 0) return null;
+ git(checkout, 'commit', '-qm', message);
+ return git(checkout, 'rev-parse', 'HEAD');
+}
+
+export async function publishReleaseGateReport({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ source,
+ runId,
+ maxAttempts = 5,
+ beforePush,
+}) {
+ const sourceDir = path.resolve(source);
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+ const checkout = checkoutPages({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ sparsePaths: [path.join('visual-gate')],
+ prefix: 'gh-pages-release-gate-',
+ });
+ try {
+ const runDir = path.join(checkout, 'visual-gate', String(runId));
+ const latestDir = path.join(checkout, 'visual-gate', 'latest');
+ fs.rmSync(runDir, {recursive: true, force: true});
+ fs.rmSync(latestDir, {recursive: true, force: true});
+ copyContents(sourceDir, runDir);
+ copyContents(sourceDir, latestDir);
+ pruneReleaseGateRuns(path.join(checkout, 'visual-gate'));
+ const commit = commitIfNeeded(checkout, `release gate: ${runId}`, [
+ '-A',
+ 'visual-gate',
+ ]);
+ if (commit === null) {
+ process.stdout.write('Nothing to publish.\n');
+ return {published: false};
+ }
+ await beforePush?.({attempt, checkout, commit});
+ const pushed = tryRun('git', [
+ '-C',
+ checkout,
+ 'push',
+ 'origin',
+ PAGES_BRANCH,
+ ]);
+ if (pushOrRetry(pushed)) {
+ process.stdout.write(
+ `Published https://facebook.github.io/astryx/visual-gate/${runId}/\n`,
+ );
+ return {published: true, commit};
+ }
+ } finally {
+ fs.rmSync(checkout, {recursive: true, force: true});
+ }
+ process.stdout.write(
+ `Push rejected; retrying release-gate publish (${attempt}/${maxAttempts}).\n`,
+ );
+ await sleep(attempt * 2000);
+ }
+ refuse(
+ `could not publish the release-gate report after ${maxAttempts} attempts`,
+ );
+}
+
+function pruneReleaseGateRuns(visualGateRoot) {
+ if (!fs.existsSync(visualGateRoot)) return;
+ const runs = fs
+ .readdirSync(visualGateRoot, {withFileTypes: true})
+ .filter(entry => entry.isDirectory() && RUN_ID.test(entry.name))
+ .map(entry => Number(entry.name))
+ .sort((a, b) => a - b);
+ for (const runId of runs.slice(0, Math.max(0, runs.length - 20))) {
+ fs.rmSync(path.join(visualGateRoot, String(runId)), {
+ recursive: true,
+ force: true,
+ });
+ }
+}
+
+export async function publishStableSite({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ source,
+ sha,
+ maxAttempts = 5,
+ beforePush,
+}) {
+ const sourceDir = path.resolve(source);
+ const shortSha = sha.slice(0, 7);
+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
+ const checkout = checkoutPages({
+ repository,
+ token,
+ tempRoot,
+ remoteURL,
+ noCheckout: true,
+ sparsePaths: ['storybook', 'sandbox', 'assets'],
+ prefix: 'gh-pages-stable-',
+ });
+ try {
+ const oldHead = git(checkout, 'rev-parse', 'HEAD');
+ for (const rel of [
+ 'storybook',
+ 'sandbox',
+ 'assets',
+ 'index.html',
+ 'latest',
+ ]) {
+ fs.rmSync(path.join(checkout, rel), {recursive: true, force: true});
+ }
+ copyContents(sourceDir, checkout);
+ fs.closeSync(fs.openSync(path.join(checkout, '.nojekyll'), 'a'));
+ git(checkout, 'add', '-A');
+ const diff = tryRun('git', [
+ '-C',
+ checkout,
+ 'diff',
+ '--cached',
+ '--quiet',
+ ]);
+ if (diff.status === 0) {
+ process.stdout.write('Nothing to publish.\n');
+ return {published: false};
+ }
+ configureGitIdentity(checkout);
+ const tree = git(checkout, 'write-tree');
+ const commit = run('git', [
+ '-C',
+ checkout,
+ 'commit-tree',
+ tree,
+ '-m',
+ `Deploy ${shortSha} to GitHub Pages`,
+ ]);
+ await beforePush?.({attempt, checkout, commit, oldHead});
+ const pushed = tryRun('git', [
+ '-C',
+ checkout,
+ 'push',
+ 'origin',
+ `--force-with-lease=refs/heads/${PAGES_BRANCH}:${oldHead}`,
+ `${commit}:refs/heads/${PAGES_BRANCH}`,
+ ]);
+ if (pushOrRetry(pushed)) {
+ process.stdout.write(
+ `Published stable site ${shortSha} (${commit.slice(0, 7)}).\n`,
+ );
+ return {published: true, commit};
+ }
+ } finally {
+ fs.rmSync(checkout, {recursive: true, force: true});
+ }
+ process.stdout.write(
+ `Push rejected; retrying stable-site publish (${attempt}/${maxAttempts}).\n`,
+ );
+ await sleep(attempt * 2000);
+ }
+ refuse(`could not publish the stable site after ${maxAttempts} attempts`);
+}
+
+export async function withPublicationTurn({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ publish,
+}) {
+ validateIdentity(repository, runId, scope);
+ await enqueuePublication({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ await waitForPublicationTurn({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ try {
+ return await publish();
+ } finally {
+ await releasePublication({
+ repository,
+ runId,
+ scope,
+ token,
+ tempRoot,
+ remoteURL,
+ });
+ }
+}
+
+function flag(args, name, fallback) {
+ const index = args.indexOf(name);
+ if (index === -1) return fallback;
+ const value = args[index + 1];
+ if (!value || value.startsWith('--')) refuse(`${name} requires a value`);
+ return value;
+}
+
+function cliContext(args) {
+ const repository = process.env.GITHUB_REPOSITORY ?? '';
+ const runId = Number(process.env.GITHUB_RUN_ID);
+ const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN ?? '';
+ const tempRoot = process.env.RUNNER_TEMP || os.tmpdir();
+ const maxAttempts = Number(flag(args, '--max-attempts', '5'));
+ if (!token) refuse('GitHub token is missing');
+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {
+ refuse('--max-attempts is invalid');
+ }
+ return {repository, runId, token, tempRoot, maxAttempts};
+}
+
+export async function main(argv = process.argv.slice(2)) {
+ const command = argv[0];
+ const context = cliContext(argv);
+ if (command === 'stable-site') {
+ const source = flag(argv, '--source');
+ const sha = flag(argv, '--sha', process.env.GITHUB_SHA ?? '');
+ if (!source) refuse('--source is required');
+ if (!/^[0-9a-f]{7,40}$/i.test(sha)) refuse('--sha is invalid');
+ await withPublicationTurn({
+ ...context,
+ scope: 'whole-tree',
+ publish: () => publishStableSite({...context, source, sha}),
+ });
+ } else if (command === 'release-gate') {
+ const source = flag(argv, '--source');
+ if (!source) refuse('--source is required');
+ await withPublicationTurn({
+ ...context,
+ scope: 'visual-gate/reports',
+ publish: () => publishReleaseGateReport({...context, source}),
+ });
+ } else {
+ refuse(
+ 'usage: gh-pages-publisher.mjs <stable-site|release-gate> --source <dir>',
+ );
+ }
+}
+
+if (
+ process.argv[1] &&
+ fs.realpathSync(process.argv[1]) ===
+ fs.realpathSync(fileURLToPath(import.meta.url))
+) {
+ try {
+ await main();
+ } catch (error) {
+ console.error(error.message);
+ process.exitCode = 1;
+ }
+}
diff --git a/.github/scripts/lib/gh-pages-publisher.test.mjs b/.github/scripts/lib/gh-pages-publisher.test.mjs
new file mode 100644
index 0000000000..e10ad3e210
--- /dev/null
+++ b/.github/scripts/lib/gh-pages-publisher.test.mjs
@@ -0,0 +1,336 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+import {execFileSync, spawnSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {afterEach, describe, expect, it} from 'vitest';
+
+import {
+ enqueuePublication,
+ publishReleaseGateReport,
+ publishStableSite,
+ releasePublication,
+ waitForPublicationTurn,
+ withPublicationTurn,
+} from './gh-pages-publisher.mjs';
+
+const REPO = 'facebook/astryx';
+const roots = [];
+
+function git(cwd, ...args) {
+ return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim();
+}
+
+afterEach(() => {
+ for (const root of roots.splice(0)) {
+ fs.rmSync(root, {recursive: true, force: true});
+ }
+});
+
+function writeFile(file, value) {
+ fs.mkdirSync(path.dirname(file), {recursive: true});
+ fs.writeFileSync(file, value);
+}
+
+function fixture() {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gh-pages-publisher-'));
+ roots.push(root);
+ const seed = path.join(root, 'seed');
+ const remote = path.join(root, 'remote.git');
+ const bin = path.join(root, 'bin');
+ fs.mkdirSync(seed);
+ fs.mkdirSync(bin);
+ git(seed, 'init', '-q', '-b', 'gh-pages');
+ git(seed, 'config', 'user.name', 'Test');
+ git(seed, 'config', 'user.email', '[email protected]');
+ writeFile(path.join(seed, 'storybook', 'old.html'), 'old storybook');
+ writeFile(path.join(seed, 'sandbox', 'old.html'), 'old sandbox');
+ writeFile(path.join(seed, 'assets', 'old.css'), 'old asset');
+ writeFile(path.join(seed, 'pr', '123', 'index.html'), 'preview');
+ writeFile(path.join(seed, 'reports', 'vibe', 'index.html'), 'vibe');
+ writeFile(
+ path.join(seed, 'visual-gate', 'baseline', 'manifest.json'),
+ '{}\n',
+ );
+ writeFile(path.join(seed, 'visual-gate', '55', 'old.html'), 'old report');
+ writeFile(path.join(seed, 'index.html'), 'old landing');
+ writeFile(path.join(seed, 'latest'), 'oldsha');
+ writeFile(path.join(seed, '.nojekyll'), '');
+ git(seed, 'add', '.');
+ git(seed, 'commit', '-qm', 'seed');
+ git(root, 'clone', '-q', '--bare', seed, remote);
+ writeFile(
+ path.join(bin, 'gh'),
+ '#!/bin/sh\nprintf \'{"status":"in_progress"}\\n\'\n',
+ );
+ fs.chmodSync(path.join(bin, 'gh'), 0o755);
+ return {root, remote, bin};
+}
+
+function cloneRemote(remote, root, name = 'final') {
+ const checkout = path.join(root, name);
+ git(root, 'clone', '-q', '--branch', 'gh-pages', remote, checkout);
+ return checkout;
+}
+
+function context({root, remote, bin}, runId, scope) {
+ return {
+ repository: REPO,
+ runId,
+ scope,
+ token: 'test-token',
+ tempRoot: root,
+ remoteURL: `file://${remote}`,
+ envPath: `${bin}:${process.env.PATH}`,
+ };
+}
+
+async function queuedPublish(fx, runId, scope, publish) {
+ const previousPath = process.env.PATH;
+ process.env.PATH = `${fx.bin}:${previousPath}`;
+ try {
+ return await withPublicationTurn({
+ ...context(fx, runId, scope),
+ publish,
+ });
+ } finally {
+ process.env.PATH = previousPath;
+ }
+}
+
+async function withoutGitIdentity(callback) {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'git-no-identity-'));
+ const emptyGlobalConfig = path.join(root, 'config');
+ fs.writeFileSync(emptyGlobalConfig, '');
+ const keys = [
+ 'GIT_AUTHOR_NAME',
+ 'GIT_AUTHOR_EMAIL',
+ 'GIT_COMMITTER_NAME',
+ 'GIT_COMMITTER_EMAIL',
+ 'GIT_CONFIG_GLOBAL',
+ 'GIT_CONFIG_NOSYSTEM',
+ 'HOME',
+ ];
+ const previous = Object.fromEntries(keys.map(key => [key, process.env[key]]));
+ for (const key of keys) delete process.env[key];
+ process.env.GIT_CONFIG_GLOBAL = emptyGlobalConfig;
+ process.env.GIT_CONFIG_NOSYSTEM = '1';
+ process.env.HOME = root;
+ try {
+ return await callback();
+ } finally {
+ for (const key of keys) {
+ if (previous[key] === undefined) delete process.env[key];
+ else process.env[key] = previous[key];
+ }
+ fs.rmSync(root, {recursive: true, force: true});
+ }
+}
+
+describe('gh-pages publisher', () => {
+ it('queues whole-tree and scoped writers without Actions pending-run cancellation', async () => {
+ const fx = fixture();
+ const wholeTree = context(fx, 800, 'whole-tree');
+ const scoped = context(fx, 801, 'visual-gate/reports');
+ const previousPath = process.env.PATH;
+ process.env.PATH = `${fx.bin}:${previousPath}`;
+ try {
+ await enqueuePublication(wholeTree);
+ await enqueuePublication(scoped);
+ await expect(
+ waitForPublicationTurn({...scoped, timeoutMs: 0}),
+ ).rejects.toThrow(/timed out behind gh-pages publication run 800/);
+ await waitForPublicationTurn(wholeTree);
+ await releasePublication(wholeTree);
+ await waitForPublicationTurn(scoped);
+ await releasePublication(scoped);
+ } finally {
+ process.env.PATH = previousPath;
+ }
+
+ const final = cloneRemote(fx.remote, fx.root);
+ const queue = path.join(final, '.astryx-gh-pages', 'publication-queue');
+ expect(fs.existsSync(queue) ? fs.readdirSync(queue).sort() : []).toEqual(
+ [],
+ );
+ });
+
+ it('publishes the stable site as an orphan while preserving unrelated scoped paths', async () => {
+ const fx = fixture();
+ const staged = path.join(fx.root, 'staged');
+ writeFile(path.join(staged, 'storybook', 'index.html'), 'new storybook');
+ writeFile(path.join(staged, 'sandbox', 'index.html'), 'new sandbox');
+ writeFile(path.join(staged, 'assets', 'astryx.css'), 'new css');
+ writeFile(path.join(staged, 'index.html'), 'new landing');
+ writeFile(path.join(staged, 'latest'), 'abcdef1');
+ const queueWriter = cloneRemote(fx.remote, fx.root, 'queue-writer');
+ git(queueWriter, 'config', 'user.name', 'Test');
+ git(queueWriter, 'config', 'user.email', '[email protected]');
+ writeFile(
+ path.join(
+ queueWriter,
+ '.astryx-gh-pages',
+ 'publication-queue',
+ '700.json',
+ ),
+ '{"version":1,"repository":"facebook/astryx","runId":700,"scope":"whole-tree"}\n',
+ );
+ git(queueWriter, 'add', '.');
+ git(queueWriter, 'commit', '-qm', 'seed queue');
+ git(queueWriter, 'push', '-q', 'origin', 'gh-pages');
+
+ await withoutGitIdentity(() =>
+ publishStableSite({
+ ...context(fx, 900, 'whole-tree'),
+ source: staged,
+ sha: 'abcdef1234567890',
+ }),
+ );
+
+ const final = cloneRemote(fx.remote, fx.root);
+ expect(
+ fs.readFileSync(path.join(final, 'storybook', 'index.html'), 'utf8'),
+ ).toBe('new storybook');
+ expect(fs.existsSync(path.join(final, 'storybook', 'old.html'))).toBe(
+ false,
+ );
+ expect(
+ fs.readFileSync(path.join(final, 'pr', '123', 'index.html'), 'utf8'),
+ ).toBe('preview');
+ expect(
+ fs.readFileSync(
+ path.join(final, 'reports', 'vibe', 'index.html'),
+ 'utf8',
+ ),
+ ).toBe('vibe');
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
+ 'utf8',
+ ),
+ ).toBe('{}\n');
+ expect(
+ fs.readFileSync(
+ path.join(final, '.astryx-gh-pages', 'publication-queue', '700.json'),
+ 'utf8',
+ ),
+ ).toContain('whole-tree');
+ expect(git(final, 'rev-list', '--count', 'HEAD')).toBe('1');
+ });
+
+ it('publishes release-gate reports without touching the stable site or baseline', async () => {
+ const fx = fixture();
+ const report = path.join(fx.root, 'report');
+ writeFile(path.join(report, 'index.html'), 'new report');
+ writeFile(
+ path.join(report, 'release-gate.json'),
+ '{"visual":{"status":"passed"}}\n',
+ );
+
+ await publishReleaseGateReport({
+ ...context(fx, 901, 'visual-gate/reports'),
+ source: report,
+ runId: 901,
+ });
+
+ const final = cloneRemote(fx.remote, fx.root);
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate', '901', 'index.html'),
+ 'utf8',
+ ),
+ ).toBe('new report');
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate', 'latest', 'index.html'),
+ 'utf8',
+ ),
+ ).toBe('new report');
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate', 'baseline', 'manifest.json'),
+ 'utf8',
+ ),
+ ).toBe('{}\n');
+ expect(
+ fs.readFileSync(path.join(final, 'storybook', 'old.html'), 'utf8'),
+ ).toBe('old storybook');
+ });
+
+ it('retries from the source checkout after deleting the rejected gh-pages checkout', async () => {
+ const fx = fixture();
+ const report = path.join(fx.root, 'report');
+ writeFile(path.join(report, 'index.html'), 'retry report');
+ let raced = false;
+
+ await publishReleaseGateReport({
+ ...context(fx, 902, 'visual-gate/reports'),
+ source: report,
+ runId: 902,
+ beforePush: async ({attempt}) => {
+ if (attempt !== 1 || raced) return;
+ raced = true;
+ const writer = cloneRemote(fx.remote, fx.root, 'race-writer');
+ git(writer, 'config', 'user.name', 'Test');
+ git(writer, 'config', 'user.email', '[email protected]');
+ writeFile(path.join(writer, 'visual-gate', 'race.txt'), 'race');
+ git(writer, 'add', '.');
+ git(writer, 'commit', '-qm', 'race');
+ git(writer, 'push', '-q', 'origin', 'gh-pages');
+ },
+ });
+
+ expect(raced).toBe(true);
+ const final = cloneRemote(fx.remote, fx.root);
+ expect(
+ fs.readFileSync(path.join(final, 'visual-gate', 'race.txt'), 'utf8'),
+ ).toBe('race');
+ expect(
+ fs.readFileSync(
+ path.join(final, 'visual-gate', '902', 'index.html'),
+ 'utf8',
+ ),
+ ).toBe('retry report');
+ });
+
+ it('prunes only old release-gate run directories', async () => {
+ const fx = fixture();
+ const seed = cloneRemote(fx.remote, fx.root, 'seed-more-runs');
+ git(seed, 'config', 'user.name', 'Test');
+ git(seed, 'config', 'user.email', '[email protected]');
+ for (let runId = 100; runId <= 125; runId += 1) {
+ writeFile(
+ path.join(seed, 'visual-gate', String(runId), 'index.html'),
+ String(runId),
+ );
+ }
+ writeFile(
+ path.join(seed, '.astryx-gh-pages', 'publication-queue', '700.json'),
+ '{"version":1,"repository":"facebook/astryx","runId":700,"scope":"whole-tree"}\n',
+ );
+ git(seed, 'add', '.');
+ git(seed, 'commit', '-qm', 'seed runs');
+ git(seed, 'push', '-q', 'origin', 'gh-pages');
+ const report = path.join(fx.root, 'report-prune');
+ writeFile(path.join(report, 'index.html'), 'new');
+
+ await publishReleaseGateReport({
+ ...context(fx, 126, 'visual-gate/reports'),
+ source: report,
+ runId: 126,
+ });
+
+ const final = cloneRemote(fx.remote, fx.root);
+ expect(fs.existsSync(path.join(final, 'visual-gate', '100'))).toBe(false);
+ expect(fs.existsSync(path.join(final, 'visual-gate', '107'))).toBe(true);
+ expect(fs.existsSync(path.join(final, 'visual-gate', 'baseline'))).toBe(
+ true,
+ );
+ expect(fs.existsSync(path.join(final, 'visual-gate', 'latest'))).toBe(true);
+ expect(
+ fs.existsSync(path.join(final, '.astryx-gh-pages', 'publication-queue')),
+ ).toBe(true);
+ });
+});
diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
index 17570ca81b..fec47d782c 100644
--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs
+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs
@@ -231,13 +231,53 @@ describe('visual acceptance workflow concurrency', () => {
manual.indexOf('Wait for the baseline publication turn'),
).toBeLessThan(manual.indexOf('Fetch the current baseline'));
expect(workflow('release-gate.yml')).toContain(
- "grep -Ev '/(baseline|latest|publication-queue)/$'",
+ 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',
);
expect(value).toContain(
"context.eventName === 'workflow_dispatch' ? 'true' : 'false'",
);
});
+ it('publishes the stable site and release gate through the shared gh-pages publisher', () => {
+ const deploy = workflow('deploy.yml');
+ const releaseGate = workflow('release-gate.yml');
+
+ expect(deploy).toContain('Checkout publisher');
+ expect(deploy).toContain(
+ 'node .github/scripts/gh-pages-publisher.mjs stable-site --source staged',
+ );
+ expect(deploy).not.toContain('/tmp/ghp');
+ expect(releaseGate).toContain('Checkout publisher');
+ expect(releaseGate).toContain(
+ 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',
+ );
+ const releasePublisher = releaseGate.slice(
+ releaseGate.indexOf(' publish:'),
+ );
+ expect(releasePublisher).not.toContain('release-gate-publish');
+ expect(releasePublisher).not.toContain('/tmp/gh-pages');
+ });
+
+ it('grants queued gh-pages publishers read access to overlapping workflow runs', () => {
+ const deploy = workflow('deploy.yml');
+ const deployJob = deploy.slice(
+ deploy.indexOf(' deploy:'),
+ deploy.indexOf(' - name: Checkout publisher'),
+ );
+ const releaseGate = workflow('release-gate.yml');
+ const publishJob = releaseGate.slice(
+ releaseGate.indexOf(' publish:'),
+ releaseGate.indexOf(' - name: Checkout publisher'),
+ );
+
+ expect(deployJob).toContain('actions: read');
+ expect(deployJob).toContain('contents: write');
+ expect(deploy).toContain('gh-pages-publisher.mjs stable-site');
+ expect(publishJob).toContain('actions: read');
+ expect(publishJob).toContain('contents: write');
+ expect(releaseGate).toContain('gh-pages-publisher.mjs release-gate');
+ });
+
it('projects every known validation or publication failure from an always-running job', () => {
const value = workflow('visual-acceptance-promote.yml');
const status = value.slice(value.indexOf(' project-status:'));
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 4d924800c1..78404f1d11 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -22,11 +22,11 @@
# /index.html — landing page
# /latest — short hash of latest deploy
#
-# Publish model — cheap clean-slate via git plumbing (see "Publish" step):
+# Publish model — cheap clean-slate via the shared gh-pages publisher:
# Every deploy replaces ONLY the regenerated stable paths (storybook/, sandbox/,
# assets/, index.html, latest, .nojekyll) and carries /pr/ and /reports/ forward
# untouched, then pushes the result as a single ORPHAN commit (no history). This
-# keeps three properties at once:
+# `.github/scripts/gh-pages-publisher.mjs stable-site` keeps three properties at once:
# A. Deploy time is O(1) in preview count. Earlier this job did
# `git clone` of all of gh-pages + `cp -r` the whole /pr/ tree forward on
# every push; once previews grew to ~1GB / tens of thousands of files that
@@ -45,13 +45,13 @@ name: Deploy
on:
push:
- branches: ["main"]
+ branches: ['main']
workflow_dispatch:
permissions: {}
concurrency:
- group: "pages-deploy"
+ group: 'pages-deploy'
cancel-in-progress: false
jobs:
@@ -67,7 +67,6 @@ jobs:
- name: Check package.json exports are in sync
run: node scripts/sync-exports.js --check
-
test:
runs-on: 2-core-ubuntu-arm
permissions:
@@ -207,8 +206,12 @@ jobs:
if: always() && !cancelled() && needs.test.result == 'success' && needs.build.result == 'success'
runs-on: ubuntu-slim
permissions:
+ actions: read
contents: write
steps:
+ - name: Checkout publisher
+ uses: actions/checkout@v7
+
- name: Download Storybook artifact
uses: actions/download-artifact@v8
with:
@@ -333,83 +336,4 @@ jobs:
- name: Publish to gh-pages
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: |
- # Carry /pr/ and /reports/ forward WITHOUT downloading or copying them
- # (>1GB / tens of thousands of files). See the header comment for the
- # full rationale:
- # --filter=blob:none → clone fetches trees but NO file contents
- # sparse-checkout → only the stable paths are materialized;
- # every other path (/pr/, /reports/,
- # /visual-gate/) stays in the index as a
- # skip-worktree entry (never written out)
- # git write-tree → rebuilds the full tree from the index, so
- # those paths are reused by SHA
- #
- # This is an allow-BY-DEFAULT mechanism: anything outside the cone
- # below survives automatically. Adding a path to the sparse-checkout
- # set makes it a MANAGED path, wiped and republished each deploy — so
- # never add a path that another workflow owns. /visual-gate/ holds the
- # visual-regression baseline (owned by visual-baseline.yml); wiping it
- # would silently disarm the release gate.
- # commit-tree (no -p) → one ORPHAN commit == force_orphan, produced
- # without a full local copy of gh-pages
- REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"
- WORK_DIR="$PWD"
- MAX_ATTEMPTS=5
-
- git config --global user.name "github-actions[bot]"
- git config --global user.email "github-actions[bot]@users.noreply.github.com"
-
- for attempt in $(seq 1 $MAX_ATTEMPTS); do
- echo "::group::Publish attempt $attempt of $MAX_ATTEMPTS"
- cd "$WORK_DIR"
- rm -rf /tmp/ghp
-
- # Partial + no-checkout clone: fetches commits and trees, but no blobs.
- git clone --filter=blob:none --no-checkout --depth=1 \
- --single-branch --branch gh-pages "$REPO_URL" /tmp/ghp
- cd /tmp/ghp
- OLD_HEAD="$(git rev-parse HEAD)"
-
- # Materialize ONLY the stable paths (cone mode also keeps root files
- # like index.html/latest/.nojekyll). /pr/ & /reports/ are left as
- # skip-worktree index entries — never fetched, never written.
- git sparse-checkout init --cone
- git sparse-checkout set storybook sandbox assets
- git checkout gh-pages
-
- # Clean-replace the stable paths so superseded content-hashed assets
- # don't accumulate (the bug keep_files:false originally fixed).
- rm -rf storybook sandbox assets index.html latest
- cp -r "$WORK_DIR"/staged/. ./
- touch .nojekyll
-
- # git add -A respects skip-worktree, so /pr/, /reports/ and
- # /visual-gate/ are NOT staged for deletion — they stay in the
- # index and are carried forward by write-tree.
- git add -A
- if git diff --cached --quiet; then
- echo "Nothing to publish"; echo "::endgroup::"; exit 0
- fi
-
- TREE="$(git write-tree)"
- COMMIT="$(git commit-tree "$TREE" -m "Deploy ${GITHUB_SHA:0:7} to GitHub Pages")"
-
- # force-with-lease guards against a concurrent gh-pages writer
- # (another main deploy or a PR-preview push); on a lost race we
- # re-clone the fresh tip and replay, so no update is clobbered.
- if git push origin \
- "--force-with-lease=refs/heads/gh-pages:${OLD_HEAD}" \
- "${COMMIT}:refs/heads/gh-pages"; then
- echo "Published on attempt $attempt (orphan ${COMMIT:0:7})"
- echo "::endgroup::"
- exit 0
- fi
-
- echo "Push rejected (concurrent gh-pages update) — retrying in $((attempt * 2))s"
- echo "::endgroup::"
- sleep $((attempt * 2))
- done
-
- echo "::error::Failed to publish to gh-pages after ${MAX_ATTEMPTS} attempts"
- exit 1
+ run: node .github/scripts/gh-pages-publisher.mjs stable-site --source staged
diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml
index 96ad0bcfe7..b5f184e5fe 100644
--- a/.github/workflows/release-gate.yml
+++ b/.github/workflows/release-gate.yml
@@ -247,14 +247,12 @@ jobs:
if: ${{ !cancelled() }}
runs-on: ubuntu-slim
permissions:
+ actions: read
contents: write
- # Not the pages-deploy group: GitHub keeps only the newest pending run per
- # group, so joining it could cancel a pending site deploy (see
- # cleanup-previews.yml for the same reasoning).
- concurrency:
- group: release-gate-publish
- cancel-in-progress: false
steps:
+ - name: Checkout publisher
+ uses: actions/checkout@v7
+
- name: Download report
uses: actions/download-artifact@v8
with:
@@ -284,39 +282,9 @@ jobs:
cat report/release-gate.json
- name: Publish to gh-pages
- run: |
- set -eu
- REPO_URL="https://x-access-token:${{ github.token }}@github.com/${GITHUB_REPOSITORY}.git"
- for attempt in 1 2 3; do
- rm -rf /tmp/gh-pages
- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages "$REPO_URL" /tmp/gh-pages
- git -C /tmp/gh-pages sparse-checkout set visual-gate
- RUN_DIR="/tmp/gh-pages/visual-gate/${GITHUB_RUN_ID}"
- rm -rf "$RUN_DIR" /tmp/gh-pages/visual-gate/latest
- mkdir -p "$RUN_DIR" /tmp/gh-pages/visual-gate/latest
- cp -r report/. "$RUN_DIR"/
- cp -r report/. /tmp/gh-pages/visual-gate/latest/
-
- # Keep the twenty most recent run directories; the report images are
- # only useful while the change they describe is still a live question.
- ls -1d /tmp/gh-pages/visual-gate/*/ 2>/dev/null \
- | grep -Ev '/(baseline|latest|publication-queue)/$' \
- | sort -V | head -n -20 | xargs -r rm -rf
-
- cd /tmp/gh-pages
- git config user.name "github-actions[bot]"
- git config user.email "github-actions[bot]@users.noreply.github.com"
- git add -A visual-gate
- git commit -m "release gate: ${GITHUB_RUN_ID}" || { echo "Nothing to publish"; exit 0; }
- if git push origin gh-pages; then
- echo "Published https://facebook.github.io/astryx/visual-gate/${GITHUB_RUN_ID}/"
- exit 0
- fi
- echo "Push failed (attempt $attempt), retrying"
- sleep $((attempt * 10))
- done
- echo "::error::Could not publish the release-gate report to gh-pages"
- exit 1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: node .github/scripts/gh-pages-publisher.mjs release-gate --source report
- name: Summary
env: