An open source design system that's fully customizable and agent ready
TypeScript
ci: centralize gh-pages publishing (#5629)
Cindy Zhang677d68ea8f64
diff --git a/.github/scripts/gh-pages-publisher.mjs b/.github/scripts/gh-pages-publisher.mjsnew file mode 100755index 0000000000..86cc44afd4--- /dev/null+++ b/.github/scripts/gh-pages-publisher.mjs@@ -0,0 +1,6 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {main} from './lib/gh-pages-publisher.mjs';++await main();diff --git a/.github/scripts/lib/gh-pages-publisher.mjs b/.github/scripts/lib/gh-pages-publisher.mjsnew file mode 100755index 0000000000..9aa700b322--- /dev/null+++ b/.github/scripts/lib/gh-pages-publisher.mjs@@ -0,0 +1,789 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import {fileURLToPath} from 'node:url';++const PAGES_BRANCH = 'gh-pages';+const QUEUE_REL = path.join('.astryx-gh-pages', 'publication-queue');+const HOLDER_NAME = 'holder.json';+const RUN_ID = /^[1-9][0-9]*$/;+const SCOPE = /^[a-z0-9][a-z0-9._/-]*$/;+const BOT_NAME = 'github-actions[bot]';+const BOT_EMAIL = 'github-actions[bot]@users.noreply.github.com';+const DEFAULT_WAIT_MS = 75 * 60 * 1000;++function refuse(message) {+ throw new Error(`gh-pages publisher refused: ${message}`);+}++function run(command, args, options = {}) {+ const result = spawnSync(command, args, {+ encoding: 'utf8',+ ...options,+ });+ if (result.status !== 0) {+ refuse(+ `${command} ${args[0] ?? ''} failed: ${+ result.stderr?.trim() ||+ result.stdout?.trim() ||+ `exit ${result.status}`+ }`,+ );+ }+ return result.stdout.trim();+}++function tryRun(command, args, options = {}) {+ return spawnSync(command, args, {encoding: 'utf8', ...options});+}++function git(cwd, ...args) {+ return run('git', ['-C', cwd, ...args]);+}++function configureGitIdentity(cwd) {+ git(cwd, 'config', 'user.name', BOT_NAME);+ git(cwd, 'config', 'user.email', BOT_EMAIL);+}++function sleep(ms) {+ return new Promise(resolve => setTimeout(resolve, ms));+}++function validateIdentity(repository, runId, scope) {+ if (typeof repository !== 'string' || !/^[^/]+\/[^/]+$/.test(repository)) {+ refuse('repository identity is invalid');+ }+ if (!Number.isSafeInteger(runId) || runId <= 0) {+ refuse('current run id is invalid');+ }+ if (typeof scope !== 'string' || !SCOPE.test(scope) || scope.includes('..')) {+ refuse('publication scope is invalid');+ }+}++function ticketValue(repository, runId, scope) {+ return {version: 1, repository, runId, scope};+}++function validTicket(value, repository, runId, scope = value?.scope) {+ return (+ Number.isSafeInteger(runId) &&+ runId > 0 &&+ value?.version === 1 &&+ value.repository === repository &&+ value.runId === runId &&+ value.scope === scope &&+ typeof scope === 'string' &&+ SCOPE.test(scope) &&+ !scope.includes('..')+ );+}++function readJSON(file) {+ try {+ return JSON.parse(fs.readFileSync(file, 'utf8'));+ } catch {+ return null;+ }+}++export function orderedTickets(entries, repository) {+ if (!Array.isArray(entries)) refuse('queue entries are invalid');+ const tickets = [];+ const seen = new Set();+ for (const entry of entries) {+ if (!RUN_ID.test(entry?.name ?? '')) continue;+ const runId = Number(entry.name);+ if (!validTicket(entry.value, repository, runId)) {+ refuse(`queue ticket ${entry.name} has invalid identity`);+ }+ if (seen.has(runId)) refuse('queue repeats a run id');+ seen.add(runId);+ tickets.push(entry.value);+ }+ return tickets.sort((a, b) => a.runId - b.runId);+}++export function publicationBlockers(entries, repository, currentRunId, scope) {+ validateIdentity(repository, currentRunId, scope);+ const tickets = orderedTickets(entries, repository);+ if (+ !tickets.some(+ ticket => ticket.runId === currentRunId && ticket.scope === scope,+ )+ ) {+ refuse(`queue ticket for run ${currentRunId} is missing`);+ }+ return tickets.filter(ticket => ticket.runId < currentRunId);+}++export function isTerminalRun(status) {+ return status === 'completed' || status === 'missing';+}++function queueState(checkout, repository) {+ const root = path.join(checkout, QUEUE_REL);+ if (!fs.existsSync(root)) return {entries: [], invalid: [], holder: null};+ const entries = [];+ const invalid = [];+ let holder = null;+ for (const entry of fs.readdirSync(root, {withFileTypes: true})) {+ if (!entry.isFile() || !entry.name.endsWith('.json')) continue;+ const value = readJSON(path.join(root, entry.name));+ if (entry.name === HOLDER_NAME) {+ if (!validTicket(value, repository, value?.runId)) {+ refuse('publication holder has invalid identity');+ }+ holder = value;+ continue;+ }+ const name = entry.name.slice(0, -'.json'.length);+ if (!RUN_ID.test(name)) {+ invalid.push(entry.name);+ continue;+ }+ const runId = Number(name);+ if (!validTicket(value, repository, runId)) {+ invalid.push(entry.name);+ continue;+ }+ entries.push({name, value});+ }+ return {entries, invalid, holder};+}++function remoteURL({repository, token, remoteURL}) {+ return (+ remoteURL ??+ process.env.ASTRYX_GH_PAGES_REMOTE_URL ??+ `https://x-access-token:${token}@github.com/${repository}.git`+ );+}++function checkoutPages({+ repository,+ token,+ tempRoot,+ sparsePaths = [],+ noCheckout = false,+ prefix = 'gh-pages-',+ remoteURL: configuredRemoteURL,+}) {+ const checkout = fs.mkdtempSync(path.join(tempRoot, prefix));+ const args = [+ 'clone',+ '--quiet',+ '--depth=1',+ '--filter=blob:none',+ '--single-branch',+ '--branch',+ PAGES_BRANCH,+ ];+ if (noCheckout) {+ args.push('--no-checkout');+ } else if (sparsePaths.length > 0) {+ args.push('--sparse');+ }+ args.push(+ remoteURL({repository, token, remoteURL: configuredRemoteURL}),+ checkout,+ );+ run('git', args);+ if (noCheckout) {+ git(checkout, 'sparse-checkout', 'init', '--cone');+ git(checkout, 'sparse-checkout', 'set', ...sparsePaths);+ git(checkout, 'checkout', PAGES_BRANCH);+ } else if (sparsePaths.length > 0) {+ git(checkout, 'sparse-checkout', 'set', ...sparsePaths);+ }+ return checkout;+}++async function mutateQueue({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ mutate,+}) {+ validateIdentity(repository, runId, scope);+ for (let attempt = 1; attempt <= 10; attempt += 1) {+ const checkout = checkoutPages({+ repository,+ token,+ tempRoot,+ remoteURL,+ sparsePaths: [QUEUE_REL],+ prefix: 'gh-pages-queue-',+ });+ try {+ const mutation = mutate(checkout);+ if (!mutation.changed) return mutation.value;+ configureGitIdentity(checkout);+ git(checkout, 'add', '-A', QUEUE_REL);+ git(checkout, 'commit', '-qm', 'ci: update gh-pages publication queue');+ const pushed = tryRun('git', [+ '-C',+ checkout,+ 'push',+ 'origin',+ PAGES_BRANCH,+ ]);+ if (pushed.status === 0) return mutation.value;+ if (permissionDenied(pushed)) {+ refuse('push to gh-pages denied while updating the publication queue');+ }+ } finally {+ fs.rmSync(checkout, {recursive: true, force: true});+ }+ await sleep(attempt * 1000);+ }+ refuse('could not update the publication queue after 10 attempts');+}++export async function enqueuePublication(options) {+ const {repository, runId, scope} = options;+ await mutateQueue({+ ...options,+ mutate(checkout) {+ const file = path.join(checkout, QUEUE_REL, `${runId}.json`);+ if (fs.existsSync(file)) {+ if (!validTicket(readJSON(file), repository, runId, scope)) {+ refuse('existing queue ticket has invalid identity');+ }+ return {changed: false};+ }+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(+ file,+ `${JSON.stringify(ticketValue(repository, runId, scope), null, 2)}\n`,+ );+ return {changed: true};+ },+ });+ process.stdout.write(+ `Queued gh-pages publication run ${runId} (${scope}).\n`,+ );+}++export async function releasePublication(options) {+ const {repository, runId, scope} = options;+ await mutateQueue({+ ...options,+ mutate(checkout) {+ const root = path.join(checkout, QUEUE_REL);+ const ticket = path.join(root, `${runId}.json`);+ const holder = path.join(root, HOLDER_NAME);+ let changed = false;+ if (fs.existsSync(ticket)) {+ fs.rmSync(ticket);+ changed = true;+ }+ const held = readJSON(holder);+ if (held?.runId === runId && held?.scope === scope) {+ fs.rmSync(holder);+ changed = true;+ }+ return {changed};+ },+ });+ process.stdout.write(+ `Released gh-pages publication run ${runId} (${scope}).\n`,+ );+}++async function removeInvalid({+ repository,+ runId,+ scope,+ names,+ token,+ tempRoot,+ remoteURL,+}) {+ await mutateQueue({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ mutate(checkout) {+ let changed = false;+ for (const name of names) {+ const file = path.join(checkout, QUEUE_REL, name);+ if (fs.existsSync(file)) {+ fs.rmSync(file);+ changed = true;+ }+ }+ return {changed};+ },+ });+}++async function claimPublication(options) {+ const {repository, runId, scope} = options;+ return mutateQueue({+ ...options,+ mutate(checkout) {+ const state = queueState(checkout, repository);+ if (state.invalid.length > 0) return {changed: false, value: false};+ if (state.holder) {+ return {+ changed: false,+ value: state.holder.runId === runId && state.holder.scope === scope,+ };+ }+ const tickets = orderedTickets(state.entries, repository);+ if (tickets[0]?.runId !== runId || tickets[0]?.scope !== scope) {+ return {changed: false, value: false};+ }+ fs.writeFileSync(+ path.join(checkout, QUEUE_REL, HOLDER_NAME),+ `${JSON.stringify(ticketValue(repository, runId, scope), null, 2)}\n`,+ );+ return {changed: true, value: true};+ },+ });+}++function runStatus(repository, runId) {+ const result = tryRun('gh', [+ 'api',+ `repos/${repository}/actions/runs/${runId}`,+ ]);+ if (result.status === 0) {+ try {+ return JSON.parse(result.stdout).status;+ } catch (error) {+ refuse(`cannot parse queued run ${runId}: ${error.message}`);+ }+ }+ if (/HTTP 404|Not Found/i.test(result.stderr ?? '')) return 'missing';+ refuse(+ `cannot resolve queued run ${runId}: ${+ result.stderr?.trim() || result.stdout?.trim() || `exit ${result.status}`+ }`,+ );+}++export async function waitForPublicationTurn({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ timeoutMs = DEFAULT_WAIT_MS,+}) {+ validateIdentity(repository, runId, scope);+ const started = Date.now();+ const checkTimeout = blocker => {+ if (Date.now() - started >= timeoutMs) {+ refuse(+ `timed out behind gh-pages publication run ${blocker ?? 'unknown'}`,+ );+ }+ };+ while (true) {+ const checkout = checkoutPages({+ repository,+ token,+ tempRoot,+ remoteURL,+ sparsePaths: [QUEUE_REL],+ prefix: 'gh-pages-queue-',+ });+ let state;+ try {+ state = queueState(checkout, repository);+ } finally {+ fs.rmSync(checkout, {recursive: true, force: true});+ }+ if (state.invalid.length > 0) {+ checkTimeout(state.holder?.runId);+ await removeInvalid({+ repository,+ runId,+ scope,+ names: state.invalid,+ token,+ tempRoot,+ remoteURL,+ });+ continue;+ }+ if (state.holder?.runId === runId && state.holder?.scope === scope) {+ process.stdout.write(+ `gh-pages publication turn acquired by run ${runId}.\n`,+ );+ return;+ }+ if (state.holder) {+ if (isTerminalRun(runStatus(repository, state.holder.runId))) {+ checkTimeout(state.holder.runId);+ await releasePublication({+ repository,+ runId: state.holder.runId,+ scope: state.holder.scope,+ token,+ tempRoot,+ remoteURL,+ });+ continue;+ }+ } else {+ const blockers = publicationBlockers(+ state.entries,+ repository,+ runId,+ scope,+ );+ let pruned = false;+ for (const blocker of blockers) {+ if (isTerminalRun(runStatus(repository, blocker.runId))) {+ checkTimeout(blocker.runId);+ await releasePublication({+ repository,+ runId: blocker.runId,+ scope: blocker.scope,+ token,+ tempRoot,+ remoteURL,+ });+ pruned = true;+ }+ }+ if (pruned) continue;+ if (+ blockers.length === 0 &&+ (await claimPublication({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ }))+ ) {+ process.stdout.write(+ `gh-pages publication turn acquired by run ${runId}.\n`,+ );+ return;+ }+ }+ const blocker =+ state.holder?.runId ??+ orderedTickets(state.entries, repository)[0]?.runId;+ checkTimeout(blocker);+ process.stdout.write(`Waiting for gh-pages publication run ${blocker}.\n`);+ await sleep(30000);+ }+}++function copyContents(source, destination) {+ fs.mkdirSync(destination, {recursive: true});+ for (const entry of fs.readdirSync(source, {withFileTypes: true})) {+ fs.cpSync(+ path.join(source, entry.name),+ path.join(destination, entry.name),+ {+ recursive: true,+ force: true,+ },+ );+ }+}++function permissionDenied(result) {+ const text = `${result.stderr ?? ''}\n${result.stdout ?? ''}`;+ return /denied|403|permission|not authorized/i.test(text);+}++function pushOrRetry(result) {+ if (result.status === 0) return true;+ if (permissionDenied(result)) {+ refuse('push to gh-pages denied');+ }+ return false;+}++function commitIfNeeded(checkout, message, addArgs) {+ configureGitIdentity(checkout);+ git(checkout, 'add', ...addArgs);+ const diff = tryRun('git', ['-C', checkout, 'diff', '--cached', '--quiet']);+ if (diff.status === 0) return null;+ git(checkout, 'commit', '-qm', message);+ return git(checkout, 'rev-parse', 'HEAD');+}++export async function publishReleaseGateReport({+ repository,+ token,+ tempRoot,+ remoteURL,+ source,+ runId,+ maxAttempts = 5,+ beforePush,+}) {+ const sourceDir = path.resolve(source);+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {+ const checkout = checkoutPages({+ repository,+ token,+ tempRoot,+ remoteURL,+ sparsePaths: [path.join('visual-gate')],+ prefix: 'gh-pages-release-gate-',+ });+ try {+ const runDir = path.join(checkout, 'visual-gate', String(runId));+ const latestDir = path.join(checkout, 'visual-gate', 'latest');+ fs.rmSync(runDir, {recursive: true, force: true});+ fs.rmSync(latestDir, {recursive: true, force: true});+ copyContents(sourceDir, runDir);+ copyContents(sourceDir, latestDir);+ pruneReleaseGateRuns(path.join(checkout, 'visual-gate'));+ const commit = commitIfNeeded(checkout, `release gate: ${runId}`, [+ '-A',+ 'visual-gate',+ ]);+ if (commit === null) {+ process.stdout.write('Nothing to publish.\n');+ return {published: false};+ }+ await beforePush?.({attempt, checkout, commit});+ const pushed = tryRun('git', [+ '-C',+ checkout,+ 'push',+ 'origin',+ PAGES_BRANCH,+ ]);+ if (pushOrRetry(pushed)) {+ process.stdout.write(+ `Published https://facebook.github.io/astryx/visual-gate/${runId}/\n`,+ );+ return {published: true, commit};+ }+ } finally {+ fs.rmSync(checkout, {recursive: true, force: true});+ }+ process.stdout.write(+ `Push rejected; retrying release-gate publish (${attempt}/${maxAttempts}).\n`,+ );+ await sleep(attempt * 2000);+ }+ refuse(+ `could not publish the release-gate report after ${maxAttempts} attempts`,+ );+}++function pruneReleaseGateRuns(visualGateRoot) {+ if (!fs.existsSync(visualGateRoot)) return;+ const runs = fs+ .readdirSync(visualGateRoot, {withFileTypes: true})+ .filter(entry => entry.isDirectory() && RUN_ID.test(entry.name))+ .map(entry => Number(entry.name))+ .sort((a, b) => a - b);+ for (const runId of runs.slice(0, Math.max(0, runs.length - 20))) {+ fs.rmSync(path.join(visualGateRoot, String(runId)), {+ recursive: true,+ force: true,+ });+ }+}++export async function publishStableSite({+ repository,+ token,+ tempRoot,+ remoteURL,+ source,+ sha,+ maxAttempts = 5,+ beforePush,+}) {+ const sourceDir = path.resolve(source);+ const shortSha = sha.slice(0, 7);+ for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {+ const checkout = checkoutPages({+ repository,+ token,+ tempRoot,+ remoteURL,+ noCheckout: true,+ sparsePaths: ['storybook', 'sandbox', 'assets'],+ prefix: 'gh-pages-stable-',+ });+ try {+ const oldHead = git(checkout, 'rev-parse', 'HEAD');+ for (const rel of [+ 'storybook',+ 'sandbox',+ 'assets',+ 'index.html',+ 'latest',+ ]) {+ fs.rmSync(path.join(checkout, rel), {recursive: true, force: true});+ }+ copyContents(sourceDir, checkout);+ fs.closeSync(fs.openSync(path.join(checkout, '.nojekyll'), 'a'));+ git(checkout, 'add', '-A');+ const diff = tryRun('git', [+ '-C',+ checkout,+ 'diff',+ '--cached',+ '--quiet',+ ]);+ if (diff.status === 0) {+ process.stdout.write('Nothing to publish.\n');+ return {published: false};+ }+ configureGitIdentity(checkout);+ const tree = git(checkout, 'write-tree');+ const commit = run('git', [+ '-C',+ checkout,+ 'commit-tree',+ tree,+ '-m',+ `Deploy ${shortSha} to GitHub Pages`,+ ]);+ await beforePush?.({attempt, checkout, commit, oldHead});+ const pushed = tryRun('git', [+ '-C',+ checkout,+ 'push',+ 'origin',+ `--force-with-lease=refs/heads/${PAGES_BRANCH}:${oldHead}`,+ `${commit}:refs/heads/${PAGES_BRANCH}`,+ ]);+ if (pushOrRetry(pushed)) {+ process.stdout.write(+ `Published stable site ${shortSha} (${commit.slice(0, 7)}).\n`,+ );+ return {published: true, commit};+ }+ } finally {+ fs.rmSync(checkout, {recursive: true, force: true});+ }+ process.stdout.write(+ `Push rejected; retrying stable-site publish (${attempt}/${maxAttempts}).\n`,+ );+ await sleep(attempt * 2000);+ }+ refuse(`could not publish the stable site after ${maxAttempts} attempts`);+}++export async function withPublicationTurn({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ publish,+}) {+ validateIdentity(repository, runId, scope);+ await enqueuePublication({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ });+ await waitForPublicationTurn({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ });+ try {+ return await publish();+ } finally {+ await releasePublication({+ repository,+ runId,+ scope,+ token,+ tempRoot,+ remoteURL,+ });+ }+}++function flag(args, name, fallback) {+ const index = args.indexOf(name);+ if (index === -1) return fallback;+ const value = args[index + 1];+ if (!value || value.startsWith('--')) refuse(`${name} requires a value`);+ return value;+}++function cliContext(args) {+ const repository = process.env.GITHUB_REPOSITORY ?? '';+ const runId = Number(process.env.GITHUB_RUN_ID);+ const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN ?? '';+ const tempRoot = process.env.RUNNER_TEMP || os.tmpdir();+ const maxAttempts = Number(flag(args, '--max-attempts', '5'));+ if (!token) refuse('GitHub token is missing');+ if (!Number.isSafeInteger(maxAttempts) || maxAttempts <= 0) {+ refuse('--max-attempts is invalid');+ }+ return {repository, runId, token, tempRoot, maxAttempts};+}++export async function main(argv = process.argv.slice(2)) {+ const command = argv[0];+ const context = cliContext(argv);+ if (command === 'stable-site') {+ const source = flag(argv, '--source');+ const sha = flag(argv, '--sha', process.env.GITHUB_SHA ?? '');+ if (!source) refuse('--source is required');+ if (!/^[0-9a-f]{7,40}$/i.test(sha)) refuse('--sha is invalid');+ await withPublicationTurn({+ ...context,+ scope: 'whole-tree',+ publish: () => publishStableSite({...context, source, sha}),+ });+ } else if (command === 'release-gate') {+ const source = flag(argv, '--source');+ if (!source) refuse('--source is required');+ await withPublicationTurn({+ ...context,+ scope: 'visual-gate/reports',+ publish: () => publishReleaseGateReport({...context, source}),+ });+ } else {+ refuse(+ 'usage: gh-pages-publisher.mjs <stable-site|release-gate> --source <dir>',+ );+ }+}++if (+ process.argv[1] &&+ fs.realpathSync(process.argv[1]) ===+ fs.realpathSync(fileURLToPath(import.meta.url))+) {+ try {+ await main();+ } catch (error) {+ console.error(error.message);+ process.exitCode = 1;+ }+}diff --git a/.github/scripts/lib/gh-pages-publisher.test.mjs b/.github/scripts/lib/gh-pages-publisher.test.mjsnew file mode 100644index 0000000000..e10ad3e210--- /dev/null+++ b/.github/scripts/lib/gh-pages-publisher.test.mjs@@ -0,0 +1,336 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++import {execFileSync, spawnSync} from 'node:child_process';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';++import {afterEach, describe, expect, it} from 'vitest';++import {+ enqueuePublication,+ publishReleaseGateReport,+ publishStableSite,+ releasePublication,+ waitForPublicationTurn,+ withPublicationTurn,+} from './gh-pages-publisher.mjs';++const REPO = 'facebook/astryx';+const roots = [];++function git(cwd, ...args) {+ return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim();+}++afterEach(() => {+ for (const root of roots.splice(0)) {+ fs.rmSync(root, {recursive: true, force: true});+ }+});++function writeFile(file, value) {+ fs.mkdirSync(path.dirname(file), {recursive: true});+ fs.writeFileSync(file, value);+}++function fixture() {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gh-pages-publisher-'));+ roots.push(root);+ const seed = path.join(root, 'seed');+ const remote = path.join(root, 'remote.git');+ const bin = path.join(root, 'bin');+ fs.mkdirSync(seed);+ fs.mkdirSync(bin);+ git(seed, 'init', '-q', '-b', 'gh-pages');+ git(seed, 'config', 'user.name', 'Test');+ git(seed, 'config', 'user.email', '[email protected]');+ writeFile(path.join(seed, 'storybook', 'old.html'), 'old storybook');+ writeFile(path.join(seed, 'sandbox', 'old.html'), 'old sandbox');+ writeFile(path.join(seed, 'assets', 'old.css'), 'old asset');+ writeFile(path.join(seed, 'pr', '123', 'index.html'), 'preview');+ writeFile(path.join(seed, 'reports', 'vibe', 'index.html'), 'vibe');+ writeFile(+ path.join(seed, 'visual-gate', 'baseline', 'manifest.json'),+ '{}\n',+ );+ writeFile(path.join(seed, 'visual-gate', '55', 'old.html'), 'old report');+ writeFile(path.join(seed, 'index.html'), 'old landing');+ writeFile(path.join(seed, 'latest'), 'oldsha');+ writeFile(path.join(seed, '.nojekyll'), '');+ git(seed, 'add', '.');+ git(seed, 'commit', '-qm', 'seed');+ git(root, 'clone', '-q', '--bare', seed, remote);+ writeFile(+ path.join(bin, 'gh'),+ '#!/bin/sh\nprintf \'{"status":"in_progress"}\\n\'\n',+ );+ fs.chmodSync(path.join(bin, 'gh'), 0o755);+ return {root, remote, bin};+}++function cloneRemote(remote, root, name = 'final') {+ const checkout = path.join(root, name);+ git(root, 'clone', '-q', '--branch', 'gh-pages', remote, checkout);+ return checkout;+}++function context({root, remote, bin}, runId, scope) {+ return {+ repository: REPO,+ runId,+ scope,+ token: 'test-token',+ tempRoot: root,+ remoteURL: `file://${remote}`,+ envPath: `${bin}:${process.env.PATH}`,+ };+}++async function queuedPublish(fx, runId, scope, publish) {+ const previousPath = process.env.PATH;+ process.env.PATH = `${fx.bin}:${previousPath}`;+ try {+ return await withPublicationTurn({+ ...context(fx, runId, scope),+ publish,+ });+ } finally {+ process.env.PATH = previousPath;+ }+}++async function withoutGitIdentity(callback) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'git-no-identity-'));+ const emptyGlobalConfig = path.join(root, 'config');+ fs.writeFileSync(emptyGlobalConfig, '');+ const keys = [+ 'GIT_AUTHOR_NAME',+ 'GIT_AUTHOR_EMAIL',+ 'GIT_COMMITTER_NAME',+ 'GIT_COMMITTER_EMAIL',+ 'GIT_CONFIG_GLOBAL',+ 'GIT_CONFIG_NOSYSTEM',+ 'HOME',+ ];+ const previous = Object.fromEntries(keys.map(key => [key, process.env[key]]));+ for (const key of keys) delete process.env[key];+ process.env.GIT_CONFIG_GLOBAL = emptyGlobalConfig;+ process.env.GIT_CONFIG_NOSYSTEM = '1';+ process.env.HOME = root;+ try {+ return await callback();+ } finally {+ for (const key of keys) {+ if (previous[key] === undefined) delete process.env[key];+ else process.env[key] = previous[key];+ }+ fs.rmSync(root, {recursive: true, force: true});+ }+}++describe('gh-pages publisher', () => {+ it('queues whole-tree and scoped writers without Actions pending-run cancellation', async () => {+ const fx = fixture();+ const wholeTree = context(fx, 800, 'whole-tree');+ const scoped = context(fx, 801, 'visual-gate/reports');+ const previousPath = process.env.PATH;+ process.env.PATH = `${fx.bin}:${previousPath}`;+ try {+ await enqueuePublication(wholeTree);+ await enqueuePublication(scoped);+ await expect(+ waitForPublicationTurn({...scoped, timeoutMs: 0}),+ ).rejects.toThrow(/timed out behind gh-pages publication run 800/);+ await waitForPublicationTurn(wholeTree);+ await releasePublication(wholeTree);+ await waitForPublicationTurn(scoped);+ await releasePublication(scoped);+ } finally {+ process.env.PATH = previousPath;+ }++ const final = cloneRemote(fx.remote, fx.root);+ const queue = path.join(final, '.astryx-gh-pages', 'publication-queue');+ expect(fs.existsSync(queue) ? fs.readdirSync(queue).sort() : []).toEqual(+ [],+ );+ });++ it('publishes the stable site as an orphan while preserving unrelated scoped paths', async () => {+ const fx = fixture();+ const staged = path.join(fx.root, 'staged');+ writeFile(path.join(staged, 'storybook', 'index.html'), 'new storybook');+ writeFile(path.join(staged, 'sandbox', 'index.html'), 'new sandbox');+ writeFile(path.join(staged, 'assets', 'astryx.css'), 'new css');+ writeFile(path.join(staged, 'index.html'), 'new landing');+ writeFile(path.join(staged, 'latest'), 'abcdef1');+ const queueWriter = cloneRemote(fx.remote, fx.root, 'queue-writer');+ git(queueWriter, 'config', 'user.name', 'Test');+ git(queueWriter, 'config', 'user.email', '[email protected]');+ writeFile(+ path.join(+ queueWriter,+ '.astryx-gh-pages',+ 'publication-queue',+ '700.json',+ ),+ '{"version":1,"repository":"facebook/astryx","runId":700,"scope":"whole-tree"}\n',+ );+ git(queueWriter, 'add', '.');+ git(queueWriter, 'commit', '-qm', 'seed queue');+ git(queueWriter, 'push', '-q', 'origin', 'gh-pages');++ await withoutGitIdentity(() =>+ publishStableSite({+ ...context(fx, 900, 'whole-tree'),+ source: staged,+ sha: 'abcdef1234567890',+ }),+ );++ const final = cloneRemote(fx.remote, fx.root);+ expect(+ fs.readFileSync(path.join(final, 'storybook', 'index.html'), 'utf8'),+ ).toBe('new storybook');+ expect(fs.existsSync(path.join(final, 'storybook', 'old.html'))).toBe(+ false,+ );+ expect(+ fs.readFileSync(path.join(final, 'pr', '123', 'index.html'), 'utf8'),+ ).toBe('preview');+ expect(+ fs.readFileSync(+ path.join(final, 'reports', 'vibe', 'index.html'),+ 'utf8',+ ),+ ).toBe('vibe');+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate', 'baseline', 'manifest.json'),+ 'utf8',+ ),+ ).toBe('{}\n');+ expect(+ fs.readFileSync(+ path.join(final, '.astryx-gh-pages', 'publication-queue', '700.json'),+ 'utf8',+ ),+ ).toContain('whole-tree');+ expect(git(final, 'rev-list', '--count', 'HEAD')).toBe('1');+ });++ it('publishes release-gate reports without touching the stable site or baseline', async () => {+ const fx = fixture();+ const report = path.join(fx.root, 'report');+ writeFile(path.join(report, 'index.html'), 'new report');+ writeFile(+ path.join(report, 'release-gate.json'),+ '{"visual":{"status":"passed"}}\n',+ );++ await publishReleaseGateReport({+ ...context(fx, 901, 'visual-gate/reports'),+ source: report,+ runId: 901,+ });++ const final = cloneRemote(fx.remote, fx.root);+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate', '901', 'index.html'),+ 'utf8',+ ),+ ).toBe('new report');+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate', 'latest', 'index.html'),+ 'utf8',+ ),+ ).toBe('new report');+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate', 'baseline', 'manifest.json'),+ 'utf8',+ ),+ ).toBe('{}\n');+ expect(+ fs.readFileSync(path.join(final, 'storybook', 'old.html'), 'utf8'),+ ).toBe('old storybook');+ });++ it('retries from the source checkout after deleting the rejected gh-pages checkout', async () => {+ const fx = fixture();+ const report = path.join(fx.root, 'report');+ writeFile(path.join(report, 'index.html'), 'retry report');+ let raced = false;++ await publishReleaseGateReport({+ ...context(fx, 902, 'visual-gate/reports'),+ source: report,+ runId: 902,+ beforePush: async ({attempt}) => {+ if (attempt !== 1 || raced) return;+ raced = true;+ const writer = cloneRemote(fx.remote, fx.root, 'race-writer');+ git(writer, 'config', 'user.name', 'Test');+ git(writer, 'config', 'user.email', '[email protected]');+ writeFile(path.join(writer, 'visual-gate', 'race.txt'), 'race');+ git(writer, 'add', '.');+ git(writer, 'commit', '-qm', 'race');+ git(writer, 'push', '-q', 'origin', 'gh-pages');+ },+ });++ expect(raced).toBe(true);+ const final = cloneRemote(fx.remote, fx.root);+ expect(+ fs.readFileSync(path.join(final, 'visual-gate', 'race.txt'), 'utf8'),+ ).toBe('race');+ expect(+ fs.readFileSync(+ path.join(final, 'visual-gate', '902', 'index.html'),+ 'utf8',+ ),+ ).toBe('retry report');+ });++ it('prunes only old release-gate run directories', async () => {+ const fx = fixture();+ const seed = cloneRemote(fx.remote, fx.root, 'seed-more-runs');+ git(seed, 'config', 'user.name', 'Test');+ git(seed, 'config', 'user.email', '[email protected]');+ for (let runId = 100; runId <= 125; runId += 1) {+ writeFile(+ path.join(seed, 'visual-gate', String(runId), 'index.html'),+ String(runId),+ );+ }+ writeFile(+ path.join(seed, '.astryx-gh-pages', 'publication-queue', '700.json'),+ '{"version":1,"repository":"facebook/astryx","runId":700,"scope":"whole-tree"}\n',+ );+ git(seed, 'add', '.');+ git(seed, 'commit', '-qm', 'seed runs');+ git(seed, 'push', '-q', 'origin', 'gh-pages');+ const report = path.join(fx.root, 'report-prune');+ writeFile(path.join(report, 'index.html'), 'new');++ await publishReleaseGateReport({+ ...context(fx, 126, 'visual-gate/reports'),+ source: report,+ runId: 126,+ });++ const final = cloneRemote(fx.remote, fx.root);+ expect(fs.existsSync(path.join(final, 'visual-gate', '100'))).toBe(false);+ expect(fs.existsSync(path.join(final, 'visual-gate', '107'))).toBe(true);+ expect(fs.existsSync(path.join(final, 'visual-gate', 'baseline'))).toBe(+ true,+ );+ expect(fs.existsSync(path.join(final, 'visual-gate', 'latest'))).toBe(true);+ expect(+ fs.existsSync(path.join(final, '.astryx-gh-pages', 'publication-queue')),+ ).toBe(true);+ });+});diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjsindex 17570ca81b..fec47d782c 100644--- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs+++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs@@ -231,13 +231,53 @@ describe('visual acceptance workflow concurrency', () => {manual.indexOf('Wait for the baseline publication turn'),).toBeLessThan(manual.indexOf('Fetch the current baseline'));expect(workflow('release-gate.yml')).toContain(- "grep -Ev '/(baseline|latest|publication-queue)/$'",+ 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',);expect(value).toContain("context.eventName === 'workflow_dispatch' ? 'true' : 'false'",);});+ it('publishes the stable site and release gate through the shared gh-pages publisher', () => {+ const deploy = workflow('deploy.yml');+ const releaseGate = workflow('release-gate.yml');++ expect(deploy).toContain('Checkout publisher');+ expect(deploy).toContain(+ 'node .github/scripts/gh-pages-publisher.mjs stable-site --source staged',+ );+ expect(deploy).not.toContain('/tmp/ghp');+ expect(releaseGate).toContain('Checkout publisher');+ expect(releaseGate).toContain(+ 'node .github/scripts/gh-pages-publisher.mjs release-gate --source report',+ );+ const releasePublisher = releaseGate.slice(+ releaseGate.indexOf(' publish:'),+ );+ expect(releasePublisher).not.toContain('release-gate-publish');+ expect(releasePublisher).not.toContain('/tmp/gh-pages');+ });++ it('grants queued gh-pages publishers read access to overlapping workflow runs', () => {+ const deploy = workflow('deploy.yml');+ const deployJob = deploy.slice(+ deploy.indexOf(' deploy:'),+ deploy.indexOf(' - name: Checkout publisher'),+ );+ const releaseGate = workflow('release-gate.yml');+ const publishJob = releaseGate.slice(+ releaseGate.indexOf(' publish:'),+ releaseGate.indexOf(' - name: Checkout publisher'),+ );++ expect(deployJob).toContain('actions: read');+ expect(deployJob).toContain('contents: write');+ expect(deploy).toContain('gh-pages-publisher.mjs stable-site');+ expect(publishJob).toContain('actions: read');+ expect(publishJob).toContain('contents: write');+ expect(releaseGate).toContain('gh-pages-publisher.mjs release-gate');+ });+it('projects every known validation or publication failure from an always-running job', () => {const value = workflow('visual-acceptance-promote.yml');const status = value.slice(value.indexOf(' project-status:'));diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.ymlindex 4d924800c1..78404f1d11 100644--- a/.github/workflows/deploy.yml+++ b/.github/workflows/deploy.yml@@ -22,11 +22,11 @@# /index.html — landing page# /latest — short hash of latest deploy#-# Publish model — cheap clean-slate via git plumbing (see "Publish" step):+# Publish model — cheap clean-slate via the shared gh-pages publisher:# Every deploy replaces ONLY the regenerated stable paths (storybook/, sandbox/,# assets/, index.html, latest, .nojekyll) and carries /pr/ and /reports/ forward# untouched, then pushes the result as a single ORPHAN commit (no history). This-# keeps three properties at once:+# `.github/scripts/gh-pages-publisher.mjs stable-site` keeps three properties at once:# A. Deploy time is O(1) in preview count. Earlier this job did# `git clone` of all of gh-pages + `cp -r` the whole /pr/ tree forward on# every push; once previews grew to ~1GB / tens of thousands of files that@@ -45,13 +45,13 @@ name: Deployon:push:- branches: ["main"]+ branches: ['main']workflow_dispatch:permissions: {}concurrency:- group: "pages-deploy"+ group: 'pages-deploy'cancel-in-progress: falsejobs:@@ -67,7 +67,6 @@ jobs:- name: Check package.json exports are in syncrun: node scripts/sync-exports.js --check-test:runs-on: 2-core-ubuntu-armpermissions:@@ -207,8 +206,12 @@ jobs:if: always() && !cancelled() && needs.test.result == 'success' && needs.build.result == 'success'runs-on: ubuntu-slimpermissions:+ actions: readcontents: writesteps:+ - name: Checkout publisher+ uses: actions/checkout@v7+- name: Download Storybook artifactuses: actions/download-artifact@v8with:@@ -333,83 +336,4 @@ jobs:- name: Publish to gh-pagesenv:GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}- run: |- # Carry /pr/ and /reports/ forward WITHOUT downloading or copying them- # (>1GB / tens of thousands of files). See the header comment for the- # full rationale:- # --filter=blob:none → clone fetches trees but NO file contents- # sparse-checkout → only the stable paths are materialized;- # every other path (/pr/, /reports/,- # /visual-gate/) stays in the index as a- # skip-worktree entry (never written out)- # git write-tree → rebuilds the full tree from the index, so- # those paths are reused by SHA- #- # This is an allow-BY-DEFAULT mechanism: anything outside the cone- # below survives automatically. Adding a path to the sparse-checkout- # set makes it a MANAGED path, wiped and republished each deploy — so- # never add a path that another workflow owns. /visual-gate/ holds the- # visual-regression baseline (owned by visual-baseline.yml); wiping it- # would silently disarm the release gate.- # commit-tree (no -p) → one ORPHAN commit == force_orphan, produced- # without a full local copy of gh-pages- REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${{ github.repository }}.git"- WORK_DIR="$PWD"- MAX_ATTEMPTS=5-- git config --global user.name "github-actions[bot]"- git config --global user.email "github-actions[bot]@users.noreply.github.com"-- for attempt in $(seq 1 $MAX_ATTEMPTS); do- echo "::group::Publish attempt $attempt of $MAX_ATTEMPTS"- cd "$WORK_DIR"- rm -rf /tmp/ghp-- # Partial + no-checkout clone: fetches commits and trees, but no blobs.- git clone --filter=blob:none --no-checkout --depth=1 \- --single-branch --branch gh-pages "$REPO_URL" /tmp/ghp- cd /tmp/ghp- OLD_HEAD="$(git rev-parse HEAD)"-- # Materialize ONLY the stable paths (cone mode also keeps root files- # like index.html/latest/.nojekyll). /pr/ & /reports/ are left as- # skip-worktree index entries — never fetched, never written.- git sparse-checkout init --cone- git sparse-checkout set storybook sandbox assets- git checkout gh-pages-- # Clean-replace the stable paths so superseded content-hashed assets- # don't accumulate (the bug keep_files:false originally fixed).- rm -rf storybook sandbox assets index.html latest- cp -r "$WORK_DIR"/staged/. ./- touch .nojekyll-- # git add -A respects skip-worktree, so /pr/, /reports/ and- # /visual-gate/ are NOT staged for deletion — they stay in the- # index and are carried forward by write-tree.- git add -A- if git diff --cached --quiet; then- echo "Nothing to publish"; echo "::endgroup::"; exit 0- fi-- TREE="$(git write-tree)"- COMMIT="$(git commit-tree "$TREE" -m "Deploy ${GITHUB_SHA:0:7} to GitHub Pages")"-- # force-with-lease guards against a concurrent gh-pages writer- # (another main deploy or a PR-preview push); on a lost race we- # re-clone the fresh tip and replay, so no update is clobbered.- if git push origin \- "--force-with-lease=refs/heads/gh-pages:${OLD_HEAD}" \- "${COMMIT}:refs/heads/gh-pages"; then- echo "Published on attempt $attempt (orphan ${COMMIT:0:7})"- echo "::endgroup::"- exit 0- fi-- echo "Push rejected (concurrent gh-pages update) — retrying in $((attempt * 2))s"- echo "::endgroup::"- sleep $((attempt * 2))- done-- echo "::error::Failed to publish to gh-pages after ${MAX_ATTEMPTS} attempts"- exit 1+ run: node .github/scripts/gh-pages-publisher.mjs stable-site --source stageddiff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.ymlindex 96ad0bcfe7..b5f184e5fe 100644--- a/.github/workflows/release-gate.yml+++ b/.github/workflows/release-gate.yml@@ -247,14 +247,12 @@ jobs:if: ${{ !cancelled() }}runs-on: ubuntu-slimpermissions:+ actions: readcontents: write- # Not the pages-deploy group: GitHub keeps only the newest pending run per- # group, so joining it could cancel a pending site deploy (see- # cleanup-previews.yml for the same reasoning).- concurrency:- group: release-gate-publish- cancel-in-progress: falsesteps:+ - name: Checkout publisher+ uses: actions/checkout@v7+- name: Download reportuses: actions/download-artifact@v8with:@@ -284,39 +282,9 @@ jobs:cat report/release-gate.json- name: Publish to gh-pages- run: |- set -eu- REPO_URL="https://x-access-token:${{ github.token }}@github.com/${GITHUB_REPOSITORY}.git"- for attempt in 1 2 3; do- rm -rf /tmp/gh-pages- git clone --depth=1 --filter=blob:none --sparse --single-branch --branch gh-pages "$REPO_URL" /tmp/gh-pages- git -C /tmp/gh-pages sparse-checkout set visual-gate- RUN_DIR="/tmp/gh-pages/visual-gate/${GITHUB_RUN_ID}"- rm -rf "$RUN_DIR" /tmp/gh-pages/visual-gate/latest- mkdir -p "$RUN_DIR" /tmp/gh-pages/visual-gate/latest- cp -r report/. "$RUN_DIR"/- cp -r report/. /tmp/gh-pages/visual-gate/latest/-- # Keep the twenty most recent run directories; the report images are- # only useful while the change they describe is still a live question.- ls -1d /tmp/gh-pages/visual-gate/*/ 2>/dev/null \- | grep -Ev '/(baseline|latest|publication-queue)/$' \- | sort -V | head -n -20 | xargs -r rm -rf-- cd /tmp/gh-pages- git config user.name "github-actions[bot]"- git config user.email "github-actions[bot]@users.noreply.github.com"- git add -A visual-gate- git commit -m "release gate: ${GITHUB_RUN_ID}" || { echo "Nothing to publish"; exit 0; }- if git push origin gh-pages; then- echo "Published https://facebook.github.io/astryx/visual-gate/${GITHUB_RUN_ID}/"- exit 0- fi- echo "Push failed (attempt $attempt), retrying"- sleep $((attempt * 10))- done- echo "::error::Could not publish the release-gate report to gh-pages"- exit 1+ env:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: node .github/scripts/gh-pages-publisher.mjs release-gate --source report- name: Summaryenv: