An open source design system that's fully customizable and agent ready
TypeScript
fix(release): decide whether a private canary-only package exists from its small latest version document (#9656)
Cindy Zhang6a9853284ecd
diff --git a/.github/scripts/release-publish-workflow.test.mjs b/.github/scripts/release-publish-workflow.test.mjsindex 04e063215b..945d1e6413 100644--- a/.github/scripts/release-publish-workflow.test.mjs+++ b/.github/scripts/release-publish-workflow.test.mjs@@ -1,6 +1,6 @@// Copyright (c) Meta Platforms, Inc. and affiliates.-import {spawnSync} from 'node:child_process';+import {spawn, spawnSync} from 'node:child_process';import fs from 'node:fs';import os from 'node:os';import path from 'node:path';@@ -61,14 +61,70 @@ const PNPM_STUB =' [ -f "$PUBLISH_LOG.once" ] || { : > "$PUBLISH_LOG.once"; echo "(409) an equivalent entry already exists in the transparency log" >&2; exit 1; };;\n' +'esac; fi\n';-function runCanaryPublish({- httpStatus,- curlExitCode = '0',+/*+ * A registry stand-in on a local port that answers as npm does. The private+ * canary-only package's `latest` version document answers each status in+ * `NATIVE_STATUSES` in turn (the last one repeats; `000` drops the+ * connection). Its dist-tags answer 401, as npm answers for a missing scoped+ * package. Other dist-tags answer 200 and other version documents 404. Each+ * request path is logged.+ */+const REGISTRY_STUB = `+import fs from 'node:fs';+import http from 'node:http';+const statuses = process.env.NATIVE_STATUSES.split(',');+let nativeReads = 0;+const server = http.createServer((request, response) => {+ const url = decodeURIComponent(request.url);+ fs.appendFileSync(process.env.REQUEST_LOG, url + '\\n');+ if (url === '/-/package/@astryxdesign/native/dist-tags') {+ response.writeHead(401).end('"Unauthorized"');+ return;+ }+ if (url === '/@astryxdesign/native/latest') {+ const status = statuses[Math.min(nativeReads, statuses.length - 1)];+ nativeReads += 1;+ if (status === '000') return request.socket.destroy();+ response.writeHead(Number(status)).end('{}');+ return;+ }+ response.writeHead(url.startsWith('/-/package/') ? 200 : 404).end('{}');+});+server.listen(0, '127.0.0.1', () => console.log(server.address().port));+`;++async function startRegistryStub(root, nativeStatuses) {+ const script = path.join(root, 'registry-stub.mjs');+ const requestLog = path.join(root, 'requests.log');+ fs.writeFileSync(script, REGISTRY_STUB);+ fs.writeFileSync(requestLog, '');+ const child = spawn(process.execPath, [script], {+ env: {+ ...process.env,+ NATIVE_STATUSES: nativeStatuses,+ REQUEST_LOG: requestLog,+ },+ stdio: ['ignore', 'pipe', 'inherit'],+ });+ const port = await new Promise((resolve, reject) => {+ child.once('error', reject);+ child.stdout.once('data', data => resolve(String(data).trim()));+ });+ return {+ url: `http://127.0.0.1:${port}`,+ requests: () => fs.readFileSync(requestLog, 'utf8'),+ stop: () => child.kill(),+ };+}++async function runCanaryPublish({+ nativeStatuses,failingPackage = '',failOnce = '',}) {const root = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-canary-publish-'));const publishLog = path.join(root, 'publish.log');+ const registry = await startRegistryStub(root, nativeStatuses);try {writeJson(root, 'packages/core/package.json', {name: '@astryxdesign/core',@@ -84,13 +140,11 @@ function runCanaryPublish({name: '@astryxdesign/theme-neutral',version: '0.7.4',});- writeExecutable(- root,- 'curl',- '#!/bin/sh\nprintf %s "$NATIVE_HTTP_STATUS"\nexit "$CURL_EXIT_CODE"\n',- );writeExecutable(root, 'pnpm', PNPM_STUB);- const publisherEnv = withPublisher(root);+ const publisherEnv = {+ ...withPublisher(root),+ ASTRYX_PUBLISH_REGISTRY: registry.url,+ };const runnerTemp = path.join(root, 'runner-temp');fs.mkdirSync(runnerTemp, {recursive: true});@@ -103,11 +157,9 @@ function runCanaryPublish({env: {...process.env,...publisherEnv,- CURL_EXIT_CODE: curlExitCode,FAIL_ONCE: failOnce,FAIL_PACKAGE: failingPackage,GITHUB_SHA: 'abcdef1234567890',- NATIVE_HTTP_STATUS: httpStatus,PATH: `${path.join(root, 'bin')}:${process.env.PATH}`,PUBLISH_LOG: publishLog,RUNNER_TEMP: runnerTemp,@@ -116,11 +168,13 @@ function runCanaryPublish({);return {...result,+ requests: registry.requests(),published: fs.existsSync(publishLog)? fs.readFileSync(publishLog, 'utf8'): '',};} finally {+ registry.stop();fs.rmSync(root, {recursive: true, force: true});}}@@ -297,8 +351,8 @@ describe('stable and canary publication authority', () => {expect(canaryText).not.toContain('release-publish-receipt');});- it('skips only an absent package that was private and canary-only', () => {- const result = runCanaryPublish({httpStatus: '404'});+ it('skips only an absent package that was private and canary-only', async () => {+ const result = await runCanaryPublish({nativeStatuses: '404'});expect(result.status).toBe(0);expect(result.stdout).toContain(@@ -309,38 +363,59 @@ describe('stable and canary publication authority', () => {expect(result.published).not.toContain('publish ./packages/native');});- it('publishes a bootstrapped private canary-only package normally', () => {- const result = runCanaryPublish({httpStatus: '200'});+ it('publishes a bootstrapped private canary-only package normally', async () => {+ const result = await runCanaryPublish({nativeStatuses: '200'});expect(result.status).toBe(0);expect(result.published).toContain('publish ./packages/native');});- it('fails on registry errors instead of treating them as package absence', () => {- const result = runCanaryPublish({httpStatus: '503'});+ it('asks whether a package exists through its latest version document, never its full document', async () => {+ const result = await runCanaryPublish({nativeStatuses: '200'});+ const requests = result.requests.split('\n');++ expect(requests).toContain('/@astryxdesign/native/latest');+ expect(requests).not.toContain('/@astryxdesign/native');+ expect(requests).not.toContain('/-/package/@astryxdesign/native/dist-tags');+ expect(step(canary, 'Publish canary versions').run).not.toContain('curl');+ });++ it('fails on registry errors instead of treating them as package absence', async () => {+ const result = await runCanaryPublish({nativeStatuses: '503'});expect(result.status).toBe(1);+ expect(result.stderr).toContain(+ 'Could not tell whether @astryxdesign/native exists on the registry after 3 lookups.',+ );expect(result.stdout).toContain(- 'Registry lookup for @astryxdesign/native returned HTTP 503',+ 'Canary publish failed for @astryxdesign/native',);+ expect(result.stdout).not.toContain('Skipping absent');expect(result.published).not.toContain('publish ./packages/native');});- it('records a failed registry request and continues later packages', () => {- const result = runCanaryPublish({httpStatus: '000', curlExitCode: '28'});+ it('records a dropped registry request and continues later packages', async () => {+ const result = await runCanaryPublish({nativeStatuses: '000'});expect(result.status).toBe(1);expect(result.stdout).toContain(- 'Registry lookup for @astryxdesign/native returned HTTP 000',+ 'Canary publish failed for @astryxdesign/native',);expect(result.published).not.toContain('publish ./packages/native');expect(result.published).toContain('publish ./packages/themes/neutral');});- it('still fails when a present package cannot publish', () => {- const result = runCanaryPublish({+ it('asks again after an unknown answer and publishes once the package is found', async () => {+ const result = await runCanaryPublish({nativeStatuses: '000,503,200'});++ expect(result.status).toBe(0);+ expect(result.published).toContain('publish ./packages/native');+ });++ it('still fails when a present package cannot publish', async () => {+ const result = await runCanaryPublish({failingPackage: 'packages/native',- httpStatus: '200',+ nativeStatuses: '200',});expect(result.status).toBe(1);@@ -363,10 +438,10 @@ describe('stable and canary publication authority', () => {);});- it('retries a canary whose first signature is rejected, and passes', () => {- const result = runCanaryPublish({+ it('retries a canary whose first signature is rejected, and passes', async () => {+ const result = await runCanaryPublish({failOnce: 'packages/themes/neutral',- httpStatus: '200',+ nativeStatuses: '200',});expect(result.status).toBe(0);diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex ae2edeae80..14c0c29a95 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -341,21 +341,12 @@ jobs:IS_PUBLISHABLE=$(node -p "const p=require('./$pkg_dir/package.json'); (!p.private || p.astryx?.canaryOnly) && p.name?.startsWith('@astryxdesign/') ? 'yes' : 'no'")if [ "$IS_PUBLISHABLE" = "yes" ]; thenNAME=$(node -p "require('./$pkg_dir/package.json').name")+ # A private canary-only package is skipped while its npm name+ # does not exist yet (exit 3); a registry that cannot say+ # whether it exists fails it, never skips it.+ SKIP_IF_ABSENT=""if grep -Fxq "$NAME" "$PRIVATE_CANARY_PACKAGES"; then- PACKAGE_URL=$(node -e "process.stdout.write(new URL(encodeURIComponent(process.argv[1]), 'https://registry.npmjs.org/').href)" "$NAME")- HTTP_STATUS=$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 30 "$PACKAGE_URL" || true)- case "$HTTP_STATUS" in- 200) ;;- 404)- echo "::notice::Skipping absent private canary-only package ${NAME}; bootstrap it and register release.yml as its trusted publisher."- continue- ;;- *)- echo "::error::Registry lookup for ${NAME} returned HTTP ${HTTP_STATUS}"- FAILED="${FAILED} ${NAME}"- continue- ;;- esac+ SKIP_IF_ABSENT="--skip-if-package-absent"fiecho "Publishing ${NAME}@${CANARY_VERSION}"# publish-package.mjs publishes at most once and lets the@@ -363,10 +354,18 @@ jobs:# failed attempt (npm's provenance signing can answer "409 an# equivalent entry already exists") is retried with a new# signature only while the registry does not have the version.- if ! node scripts/npm/publish-package.mjs --dir "./$pkg_dir" --tag canary; then- echo "::error::Canary publish failed for ${NAME}@${CANARY_VERSION}"- FAILED="${FAILED} ${NAME}"- fi+ STATUS=0+ node scripts/npm/publish-package.mjs --dir "./$pkg_dir" --tag canary $SKIP_IF_ABSENT || STATUS=$?+ case "$STATUS" in+ 0) ;;+ 3)+ echo "::notice::Skipping absent private canary-only package ${NAME}; bootstrap it and register release.yml as its trusted publisher."+ ;;+ *)+ echo "::error::Canary publish failed for ${NAME}@${CANARY_VERSION}"+ FAILED="${FAILED} ${NAME}"+ ;;+ esacfidonediff --git a/scripts/npm/publish-package.mjs b/scripts/npm/publish-package.mjsindex 78ec72e76c..f1eff6d31b 100644--- a/scripts/npm/publish-package.mjs+++ b/scripts/npm/publish-package.mjs@@ -4,8 +4,8 @@/*** @file Publish one package to npm, at most once, with retries the registry* decides.- * @input `--dir <package dir> --tag <dist-tag>`; the package's own- * package.json supplies its name and version. `ASTRYX_PUBLISH_REGISTRY`+ * @input `--dir <package dir> --tag <dist-tag> [--skip-if-package-absent]`;+ * the package's own package.json supplies its name and version. `ASTRYX_PUBLISH_REGISTRY`* (default https://registry.npmjs.org), `ASTRYX_PUBLISH_ATTEMPTS` (3) and* `ASTRYX_PUBLISH_WAITS_MS` (milliseconds to wait before each retry, default* "20000,60000"; its first value is also the wait between registry reads,@@ -13,6 +13,9 @@* @output Exit 0 when the registry has `name@version` (already there, just* published, or published by an attempt whose client reported a failure);* exit 1 when every attempt failed and the registry still does not have it.+ * With `--skip-if-package-absent`, exit 3 without publishing when the+ * package itself does not exist on the registry, and exit 1 when the+ * registry cannot say whether it exists.* @position Both publish jobs in .github/workflows/release.yml call it once* per package. The registry, not a client's exit code, decides whether a* package is published:@@ -25,6 +28,13 @@* and a new attempt signs a new entry, so it is retried.* - npm never accepts a second publish of a version, so a retry cannot* publish twice even when a registry read is stale.+ * - `--skip-if-package-absent` is for a private canary-only package whose+ * npm name may not be set up yet: its first publish would be refused.+ * The check reads the package's `latest` version document, which stays+ * small and answers 404 for a missing package, and never its full+ * document, which grows with every canary and can outlast a read's time+ * limit. (A missing scoped package's dist-tags answer 401, not 404.) An+ * unknown answer is asked again and never skips.*/import {spawnSync} from 'node:child_process';@@ -40,6 +50,8 @@ const DEFAULT_READ_INTERVAL_MS = 5_000;// Registry reads after a failed attempt: a version npm accepted can take a// few seconds to appear.const READS_AFTER_FAILURE = 4;+const PACKAGE_LOOKUPS = 3;+export const EXIT_PACKAGE_ABSENT = 3;/*** Whether the registry has `name@version`: `true`, `false`, or `null` when@@ -67,6 +79,46 @@ export async function registryHas({}}+/**+ * Whether the package `name` exists on the registry, read from its `latest`+ * version document: every published package has one. `true`, `false` (404),+ * or `null` when the read failed (a network error or any other status).+ */+export function packageExists({+ name,+ registry = DEFAULT_REGISTRY,+ fetchImpl = fetch,+}) {+ return registryHas({name, version: 'latest', registry, fetchImpl});+}++/**+ * Asks `exists()` until it gives a definite answer, at most `lookups` times,+ * waiting `intervalMs`, then twice that, between tries. Returns `true`,+ * `false`, or `null` when every answer was unknown.+ */+export async function lookUpPackage({+ name,+ exists,+ lookups = PACKAGE_LOOKUPS,+ intervalMs = DEFAULT_READ_INTERVAL_MS,+ sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),+ log = () => {},+}) {+ for (let lookup = 1; lookup <= lookups; lookup += 1) {+ const answer = await exists();+ if (answer !== null) return answer;+ if (lookup < lookups) {+ const wait = intervalMs * 2 ** (lookup - 1);+ log(+ `Could not tell whether ${name} exists on the registry (lookup ${lookup} of ${lookups}); asking again in ${Math.round(wait / 1000)} s.`,+ );+ await sleep(wait);+ }+ }+ return null;+}+/*** Publishes `name@version` at most once. `publish()` runs one publish and* returns whether its process succeeded; `isPublished()` reads the registry@@ -120,14 +172,21 @@ export async function publishOnce({return {published: false, outcome: 'failed', attempts};}+const BOOLEAN_FLAGS = new Set(['skip-if-package-absent']);+function parseArgs(argv) {const args = {};- for (let i = 0; i < argv.length; i += 2) {+ for (let i = 0; i < argv.length; i += 1) {const flag = argv[i];+ if (BOOLEAN_FLAGS.has(flag?.slice(2))) {+ args[flag.slice(2)] = true;+ continue;+ }if (!flag?.startsWith('--') || argv[i + 1] == null) {throw new Error(`expected --flag value pairs, got ${argv.join(' ')}`);}args[flag.slice(2)] = argv[i + 1];+ i += 1;}for (const required of ['dir', 'tag']) {if (!args[required]) throw new Error(`--${required} is required`);@@ -145,7 +204,8 @@ function numberList(raw, fallback) {}async function main() {- const {dir, tag} = parseArgs(process.argv.slice(2));+ const args = parseArgs(process.argv.slice(2));+ const {dir, tag} = args;const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8'),);@@ -158,12 +218,31 @@ async function main() {throw new Error('ASTRYX_PUBLISH_ATTEMPTS must be a positive integer');}const waits = numberList(process.env.ASTRYX_PUBLISH_WAITS_MS, null);+ const readIntervalMs = waits ? waits[0] : DEFAULT_READ_INTERVAL_MS;+ if (args['skip-if-package-absent']) {+ const exists = await lookUpPackage({+ name,+ intervalMs: readIntervalMs,+ exists: () => packageExists({name, registry}),+ log: message => console.log(message),+ });+ if (exists === false) {+ console.log(`${name} does not exist on the registry; not publishing it.`);+ process.exit(EXIT_PACKAGE_ABSENT);+ }+ if (exists === null) {+ console.error(+ `Could not tell whether ${name} exists on the registry after ${PACKAGE_LOOKUPS} lookups.`,+ );+ process.exit(1);+ }+ }const result = await publishOnce({name,version,attempts,retryWaitsMs: waits ?? DEFAULT_RETRY_WAITS_MS,- readIntervalMs: waits ? waits[0] : DEFAULT_READ_INTERVAL_MS,+ readIntervalMs,isPublished: () => registryHas({name, version, registry}),publish: () =>spawnSync(diff --git a/scripts/npm/publish-package.test.mjs b/scripts/npm/publish-package.test.mjsindex 7b50314b6a..a0457d8b8d 100644--- a/scripts/npm/publish-package.test.mjs+++ b/scripts/npm/publish-package.test.mjs@@ -7,7 +7,12 @@import {describe, expect, it} from 'vitest';-import {publishOnce, registryHas} from './publish-package.mjs';+import {+ lookUpPackage,+ packageExists,+ publishOnce,+ registryHas,+} from './publish-package.mjs';/*** A fake registry and publisher. `publishes` lists each attempt's outcome:@@ -168,3 +173,80 @@ describe('registryHas', () => {).toBe(null);});});++describe('packageExists', () => {+ const respond = status => async () => new Response('{}', {status});++ it("reads the package's latest version document, not its full document", async () => {+ const urls = [];+ await packageExists({+ name: '@astryxdesign/vega',+ registry: 'https://registry.example/',+ fetchImpl: async url => {+ urls.push(url);+ return new Response('{}', {status: 200});+ },+ });+ expect(urls).toEqual([+ 'https://registry.example/%40astryxdesign%2Fvega/latest',+ ]);+ });++ it('answers true for 200, false only for 404, and unknown otherwise, a 401 included', async () => {+ const name = '@astryxdesign/vega';+ expect(await packageExists({name, fetchImpl: respond(200)})).toBe(true);+ expect(await packageExists({name, fetchImpl: respond(404)})).toBe(false);+ expect(await packageExists({name, fetchImpl: respond(503)})).toBeNull();+ // npm answers 401 on a missing scoped package's dist-tags; a 401 is+ // never read as absence.+ expect(await packageExists({name, fetchImpl: respond(401)})).toBeNull();+ expect(+ await packageExists({+ name,+ fetchImpl: async () => {+ throw new Error('timed out');+ },+ }),+ ).toBeNull();+ });+});++describe('lookUpPackage', () => {+ function answers(...values) {+ const calls = {asked: 0, sleeps: []};+ return {+ calls,+ options: {+ name: '@astryxdesign/vega',+ intervalMs: 5_000,+ exists: async () => values[Math.min(calls.asked++, values.length - 1)],+ sleep: async ms => {+ calls.sleeps.push(ms);+ },+ },+ };+ }++ it('takes the first definite answer', async () => {+ const found = answers(true);+ expect(await lookUpPackage(found.options)).toBe(true);+ expect(found.calls.asked).toBe(1);++ const absent = answers(false);+ expect(await lookUpPackage(absent.options)).toBe(false);+ expect(absent.calls.asked).toBe(1);+ });++ it('asks again after an unknown answer, waiting longer each time', async () => {+ const {calls, options} = answers(null, null, false);+ expect(await lookUpPackage(options)).toBe(false);+ expect(calls.asked).toBe(3);+ expect(calls.sleeps).toEqual([5_000, 10_000]);+ });++ it('stays unknown after every lookup is unknown, never reporting absence', async () => {+ const {calls, options} = answers(null);+ expect(await lookUpPackage(options)).toBeNull();+ expect(calls.asked).toBe(3);+ });+});diff --git a/scripts/npm/registry.mjs b/scripts/npm/registry.mjsindex dbbb40fca2..31638271a0 100644--- a/scripts/npm/registry.mjs+++ b/scripts/npm/registry.mjs@@ -4,7 +4,8 @@* @file registry.mjs* @input A package name and version, an npm registry URL, and a fetch* @output `fetchVersionDocument`: the registry's response for one exact- * version's document+ * version's document (a dist-tag such as `latest` also names a version);+ * `fetchDistTags`: the response for a package's dist-tags, a small document* @position The one way the release scripts read a version from npm:* publish-package.mjs asks whether a version exists before and after each* publish, and verify-published.mjs confirms a release propagated. Keeping@@ -23,6 +24,12 @@ export function registryUrl(registry, pathname) {return new URL(pathname, `${registry.replace(/\/+$/u, '')}/`).href;}+const READ_OPTIONS = () => ({+ cache: 'no-store',+ headers: {accept: 'application/json'},+ signal: AbortSignal.timeout(30_000),+});+/** Fetches the document for exactly `name@version`, never from a cache. */export function fetchVersionDocument({name,@@ -32,10 +39,21 @@ export function fetchVersionDocument({}) {return fetchImpl(registryUrl(registry, `${packageKey(name)}/${encodeURIComponent(version)}`),- {- cache: 'no-store',- headers: {accept: 'application/json'},- signal: AbortSignal.timeout(30_000),- },+ READ_OPTIONS(),+ );+}++/**+ * Fetches a package's dist-tags, never from a cache. It stays small however+ * many versions the package has, unlike the package's full document.+ */+export function fetchDistTags({+ name,+ registry = DEFAULT_REGISTRY,+ fetchImpl = fetch,+}) {+ return fetchImpl(+ registryUrl(registry, `-/package/${packageKey(name)}/dist-tags`),+ READ_OPTIONS(),);}diff --git a/scripts/npm/verify-published.mjs b/scripts/npm/verify-published.mjsindex fd67755e7f..abc4595f22 100644--- a/scripts/npm/verify-published.mjs+++ b/scripts/npm/verify-published.mjs@@ -17,9 +17,8 @@ import {fileURLToPath} from 'node:url';import {isStableReleasePackage} from '../release/active-release.mjs';import {DEFAULT_REGISTRY,+ fetchDistTags,fetchVersionDocument,- packageKey,- registryUrl,} from './registry.mjs';const DEFAULT_ATTEMPTS = 41;@@ -43,7 +42,6 @@ export async function verifyRegistryEntry({registry = DEFAULT_REGISTRY,fetchImpl = fetch,}) {- const encoded = packageKey(name);const versionDocument = await jsonResponse(await fetchVersionDocument({name, version, registry, fetchImpl}),`${name}@${version} version document`,@@ -54,13 +52,8 @@ export async function verifyRegistryEntry({);}- const tagsUrl = registryUrl(registry, `-/package/${encoded}/dist-tags`);const tags = await jsonResponse(- await fetchImpl(tagsUrl, {- cache: 'no-store',- headers: {accept: 'application/json'},- signal: AbortSignal.timeout(30_000),- }),+ await fetchDistTags({name, registry, fetchImpl}),`${name} dist-tags`,);if (tags[tag] !== version) {