Skip to content

The open-source app everyone uses to manage agents at work

fix(release): retry verified canary tarball 404s (#15768)

Devin Foley4f680bc85c82
diff --git a/doc/RELEASING.md b/doc/RELEASING.md
index 771d3acb5b..dbf40956a5 100644
--- a/doc/RELEASING.md
+++ b/doc/RELEASING.md
@@ -104,7 +104,7 @@ It:
- publishes the user-facing `paperclipai` package last, so `paperclipai@canary` does not advance before the full package set exists
- verifies that `canary` resolves to the just-published version and that published internal dependencies exist on npm
- installs `paperclipai@canary` into a clean temporary prefix as the final npm gate
-- starts the exact published canary through a separate fresh npm install for the onboarding smoke; only npm `ETARGET` install failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline
+- starts the exact published canary through a separate fresh npm install for the onboarding smoke; npm `ETARGET` and verified canary tarball availability failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline
- fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional
- creates a git tag `canary/vYYYY.MDD.P-canary.N`
@@ -116,6 +116,23 @@ current patches and manifests while the separate lockfile-refresh PR is still
pending, without resolving new dependencies in the smoke job. The smoke still installs and tests
the exact published canary version, not a workspace build.
+The tarball retry applies only to an unambiguous npm `E404` for a canonical
+public-registry tarball of a CI-published Paperclip package at that exact canary
+version. The helper queries npm's effective global and applicable scoped registry
+in the failed install's prefix, working directory, and environment. A custom or
+unknown effective registry fails. Configuration output stays private in memory.
+Fresh primary-registry metadata must match the package, version, and tarball URL
+and announce a valid SHA512 integrity value. This establishes the announced
+artifact identity; it does not download the tarball or independently verify its
+contents. The actual npm install still checks tarball integrity.
+
+Configuration and metadata checks share a five-second deadline, including the
+response body, within the same two-minute acquisition budget. Metadata is limited
+to 64 KiB. Redirects, authentication errors, missing metadata, conflicting npm
+errors, and other artifacts fail. A persistent promised-tarball failure also
+fails after three installs. Onboarding errors never restart acquisition or
+repeat onboarding.
+
The lockfile artifact is retained for 14 days. A publisher-job rerun replaces its
source-named artifact; a smoke-only rerun uses the existing artifact. A missing
artifact fails the job rather than falling back to a different dependency set.
diff --git a/scripts/__tests__/canary-startup-fixture.mjs b/scripts/__tests__/canary-startup-fixture.mjs
index d75d1fa944..4a51ce621b 100644
--- a/scripts/__tests__/canary-startup-fixture.mjs
+++ b/scripts/__tests__/canary-startup-fixture.mjs
@@ -10,7 +10,8 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."
// Exercise the real startup helper and child-process boundary without registry
// traffic. The npm fixture installs a real executable into its isolated prefix.
-export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false }) {
+export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false,
+ config = "public", metadata = "valid", version = "2026.1009.0-canary.1" }) {
const root = mkdtempSync(path.join(os.tmpdir(), "canary-startup-test-"));
const calls = path.join(root, "calls.jsonl");
const fixtureBin = path.join(root, "bin");
@@ -39,15 +40,35 @@ if (process.env.FIXTURE_ONBOARD === "serve") {
const fs = require("node:fs");
const path = require("node:path");
const args = process.argv.slice(2);
+if (args[0] === "config") {
+ fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config", pid:process.pid, args}) + "\\n");
+ if (process.env.FIXTURE_CONFIG === "hang") {
+ const { spawn } = require("node:child_process");
+ const child = spawn(process.execPath, ["-e", "process.on('SIGTERM',()=>{}); setTimeout(()=>process.exit(0),10000)"], {stdio:"ignore"});
+ fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config-descendant", pid:child.pid}) + "\\n");
+ process.on("SIGTERM", () => {});
+ setTimeout(() => process.exit(0), 10000);
+ } else {
+ if (process.env.FIXTURE_CONFIG === "large") console.log("x".repeat(8192));
+ console.log("registry=" + (process.env.FIXTURE_CONFIG === "private" ? "https://fixture-user:[email protected]/" : "https://registry.npmjs.org/") + "\\n@paperclipai:registry=undefined");
+ console.error("fixture config diagnostic must stay private");
+ }
+} else {
const entries = fs.existsSync(process.env.FIXTURE_CALLS) ? fs.readFileSync(process.env.FIXTURE_CALLS, "utf8").trim().split("\\n").filter(Boolean).map(JSON.parse) : [];
const attempt = entries.filter(e => e.kind === "npm").length + 1;
fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"npm", pid:process.pid, args, cache:process.env.npm_config_cache}) + "\\n");
const mode = process.env.FIXTURE_MODE;
if (mode === "hang") { setTimeout(() => {}, 10000); }
else if (mode === "permanent" || (mode === "recover" && attempt === 1)) { console.error("npm error code ETARGET"); process.exit(1); }
+else if (mode === "multibyte-recover" && attempt === 1) { console.error("npm error code ETARGET\\n" + "界".repeat(30 * 1024)); process.exit(1); }
else if (mode === "auth") { console.error("npm error code E401"); process.exit(1); }
else if (mode === "mixed") { console.error("npm error code ETARGET\\nnpm error code E401"); process.exit(1); }
else if (mode === "network") { console.error("npm error code ECONNRESET"); process.exit(1); }
+else if (mode === "tarball-permanent" || mode === "tarball-large" || (mode === "tarball-recover" && attempt === 1)) {
+ const url = "https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz";
+ if (mode === "tarball-large") console.error("npm error code E401\\n" + "x".repeat(70 * 1024));
+ console.error("npm error code E404\\nnpm error 404 Not Found - GET " + url + " - Not found"); process.exit(1);
+}
else {
const prefix = args[args.indexOf("--prefix") + 1];
const bin = path.join(prefix, "node_modules", ".bin");
@@ -55,12 +76,24 @@ else {
fs.writeFileSync(path.join(bin, "paperclipai"), ${JSON.stringify(cli).replaceAll('$','\$')});
fs.chmodSync(path.join(bin, "paperclipai"), 0o755);
}
+}
`);
chmodSync(npmFixture, 0o755);
const helper = new URL("../../tests/canary-onboarding/start-published-canary.mjs", import.meta.url).href;
const driver = path.join(root, "driver.mjs");
writeFileSync(driver, `import { startPublishedCanary } from ${JSON.stringify(helper)};
-try { await startPublishedCanary({version:"2026.1009.0-canary.1", workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); }
+import { appendFileSync } from "node:fs";
+globalThis.fetch = async (url, options) => {
+ appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"metadata", url, redirect:options.redirect, credentials:options.credentials, headers:options.headers}) + "\\n");
+ const manifest = {name:"@paperclipai/server",version:process.env.PAPERCLIPAI_VERSION,dist:{tarball:"https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz",integrity:"sha512-" + Buffer.alloc(64).toString("base64")}};
+ const body = process.env.FIXTURE_METADATA === "hang" ? new ReadableStream({
+ start(controller) { options.signal.addEventListener("abort", () => controller.error(options.signal.reason), {once:true}); },
+ }) : JSON.stringify(manifest);
+ const response = new Response(body, {status:process.env.FIXTURE_METADATA === "missing" ? 404 : 200});
+ Object.defineProperty(response,"url",{value:url});
+ return response;
+};
+try { await startPublishedCanary({version:${JSON.stringify(version)}, workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); }
catch (error) { console.error(error.message); process.exitCode = 1; }
`);
const playwrightConfig = path.join(root, "playwright.config.mjs");
@@ -80,8 +113,8 @@ export default {...original, testDir:${JSON.stringify(root)}, testMatch:"fixture
? [fileURLToPath(helper), "2026.1009.0-canary.1", root, path.join(root, "data")]
: [driver], { cwd: repoRoot, env: {
...process.env, PATH: fixtureBin + path.delimiter + process.env.PATH,
- FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port),
- PAPERCLIPAI_VERSION: "2026.1009.0-canary.1", PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"),
+ FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port), FIXTURE_CONFIG: config, FIXTURE_METADATA: metadata,
+ PAPERCLIPAI_VERSION: version, PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"),
npm_config_cache: path.join(root, "fresh-cache"),
}, stdio: ["ignore", "pipe", "pipe"] });
let output = "";
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 9e5364c869..043636edb9 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -5,7 +5,9 @@ import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
+import { createServer } from "node:http";
import { canaryStartup } from "./canary-startup-fixture.mjs";
+import { canRetryPublishedTarball404, TARBALL_METADATA_TIMEOUT_MS } from "../../tests/canary-onboarding/start-published-canary.mjs";
const repoRoot = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
@@ -830,3 +832,327 @@ test("nested ETARGET output does not retry a final authentication or network fai
assert.equal(result.calls.length, 1);
}
});
+
+const exactCanary = "2026.1009.0-canary.1";
+const canaryTarball = `https://registry.npmjs.org/@paperclipai/server/-/server-${exactCanary}.tgz`;
+const tarballFailure = (url = canaryTarball, prefix = "npm error") => `${prefix} code E404\n${prefix} 404 Not Found - GET ${url} - Not found\n${prefix} 404\n${prefix} 404 The requested resource '@paperclipai/server@${url}' could not be found or you do not have permission to access it.\n`;
+
+function tarballProof({ config = {}, manifest = {}, status = 200, redirected = false, responseUrl, rawBody } = {}) {
+ const calls = [];
+ const dist = { tarball: canaryTarball, integrity: `sha512-${Buffer.alloc(64).toString("base64")}`, ...manifest.dist };
+ const data = { name: "@paperclipai/server", version: exactCanary, ...manifest, dist };
+ return { calls, options: {
+ signal: new AbortController().signal, prefix: "/fixture/install", env: {},
+ runConfig: async (command, args, options) => {
+ calls.push({ kind: "config", command, args, options });
+ return { code: 0, stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=undefined\n", ...config };
+ },
+ fetchMetadata: async (url, options) => {
+ calls.push({ kind: "metadata", url, options });
+ return { status, redirected, url: responseUrl ?? url, body: new Response(rawBody ?? JSON.stringify(data)).body };
+ },
+ } };
+}
+
+test("exact published tarball proof uses quiet effective config and fresh public metadata without credentials", async () => {
+ for (const prefix of ["npm error", "npm ERR!"]) {
+ const fixture = tarballProof();
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball, prefix), exactCanary, fixture.options), true);
+ const [config, metadata] = fixture.calls;
+ assert.deepEqual(config.args, ["config", "get", "registry", "@paperclipai:registry", "--prefix", "/fixture/install"]);
+ assert.equal(config.options.env, fixture.options.env);
+ assert.equal(config.options.quiet, true);
+ assert.equal(config.options.captureStdout, true);
+ assert.equal(config.options.captureStderr, true);
+ assert.equal(config.options.outputLimit, 4096);
+ assert.equal(metadata.url, `https://registry.npmjs.org/%40paperclipai%2Fserver/${exactCanary}`);
+ assert.equal(metadata.options.signal, config.options.signal);
+ assert.equal(metadata.options.redirect, "error");
+ assert.equal(metadata.options.credentials, "omit");
+ assert.deepEqual(metadata.options.headers, { accept: "application/json", "cache-control": "no-cache" });
+ }
+});
+
+test("mixed npm errors and noncanonical or unreleased tarballs never read config or metadata", async () => {
+ const errors = [
+ "npm error code ETARGET\n", "npm error code E401\n", "npm error code ECONNRESET\n",
+ `${tarballFailure()}npm error code E401\n`, `${tarballFailure()}npm error code 1\n`,
+ `npm error code ENEEDAUTH\n${tarballFailure()}`, `${tarballFailure()}${tarballFailure()}`,
+ `${tarballFailure()}npm error 404 Not Found - GET https://registry.example.invalid/a.tgz - Not found\n`,
+ `${tarballFailure()}npm error 404 The requested resource 'other@${canaryTarball}' could not be found\n`,
+ tarballFailure().replace(" - GET ", " - PUT "),
+ ...[
+ canaryTarball + "?token=fixture", canaryTarball + "#fragment", canaryTarball.replace("npmjs.org/", "npmjs.org:443/"),
+ canaryTarball.replace("npmjs.org/", "npmjs.org.example.invalid/"), canaryTarball.replace("npmjs.org/", "[email protected]/"),
+ canaryTarball.replace("https:", "http:"), canaryTarball.replace("@paperclipai/server", "%40paperclipai%2Fserver"),
+ canaryTarball.replace("@paperclipai/server", "@paperclipai/../server"), canaryTarball.replace(exactCanary, "2026.1009.0-canary.2"),
+ "https://registry.npmjs.org/@paperclipai/not-released/-/not-released-2026.1009.0-canary.1.tgz",
+ "https://registry.npmjs.org/@paperclipai/plugin-createos/-/plugin-createos-2026.1009.0-canary.1.tgz",
+ "https://registry.npmjs.org/%40paperclipai%2Fserver/2026.1009.0-canary.1",
+ ].map((url) => tarballFailure(url)),
+ ];
+ for (const error of errors) {
+ const fixture = tarballProof();
+ assert.equal(await canRetryPublishedTarball404(error, exactCanary, fixture.options), false, error);
+ assert.deepEqual(fixture.calls, [], error);
+ }
+ for (const version of ["2026.1009.0", "2026.1009.0-nightly.1", "2026.1009.0-beta.1", "2026.1009.0-canary.1+build"]) {
+ const fixture = tarballProof();
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball.replace(exactCanary, version)), version, fixture.options), false);
+ assert.deepEqual(fixture.calls, []);
+ }
+});
+
+test("effective registry authority respects scope precedence and rejects unknown or private config", async () => {
+ const configs = [
+ [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],
+ [{ stdout: "registry=undefined\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],
+ [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=https://registry.example.invalid/\n" }, false],
+ [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=undefined\n" }, false],
+ [{ stdout: "registry=undefined\n@paperclipai:registry=undefined\n" }, false],
+ [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=\n" }, false],
+ [{ stdout: "registry=https://fixture:[email protected]/\n@paperclipai:registry=undefined\n" }, false],
+ [{ stdout: "unexpected output\n" }, false], [{ code: 1 }, false], [{ signal: "SIGTERM" }, false], [{ outputTruncated: true }, false],
+ ];
+ for (const [config, expected] of configs) {
+ const fixture = tarballProof({ config });
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), expected);
+ assert.equal(fixture.calls.filter((call) => call.kind === "metadata").length, expected ? 1 : 0);
+ }
+});
+
+test("metadata must identify the exact published artifact with valid SHA512 syntax, not merely exist", async () => {
+ const fixtures = [
+ { status: 404 }, { status: 401 }, { status: 403 }, { status: 302 }, { redirected: true },
+ { responseUrl: "https://registry.example.invalid/metadata" }, { rawBody: "not json" },
+ { manifest: { name: "@paperclipai/shared" } }, { manifest: { version: "2026.1009.0-canary.2" } },
+ { manifest: { dist: { tarball: canaryTarball + "?query=1" } } },
+ ...[undefined, "", "sha512-A===", `sha256-${Buffer.alloc(32).toString("base64")}`, `sha512-${Buffer.alloc(64).toString("base64")} extra`]
+ .map((integrity) => ({ manifest: { dist: { integrity } } })),
+ { rawBody: JSON.stringify({ padding: "x".repeat(65 * 1024) }) },
+ ];
+ for (const input of fixtures) {
+ const fixture = tarballProof(input);
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false, JSON.stringify(input).slice(0, 200));
+ }
+});
+
+test("rejected and oversized native response bodies cancel without waiting for slow or rejected cleanup", async () => {
+ for (const [status, pending] of [[200, true], [200, false], [404, true], [404, false]]) {
+ let cancelled = false;
+ const fixture = tarballProof();
+ fixture.options.fetchMetadata = async (url) => ({ status, redirected: false, url, body: new Response(new ReadableStream({
+ start(controller) { controller.enqueue(new Uint8Array(65 * 1024)); },
+ cancel() { cancelled = true; return pending ? new Promise(() => {}) : Promise.reject(new Error("fixture rejected cancellation")); },
+ })).body });
+ const started = Date.now();
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false);
+ assert.equal(cancelled, true);
+ assert.ok(Date.now() - started < 1000, "cancellation acknowledgement must not extend a rejection");
+ }
+});
+
+test("aborting a stalled metadata read cancels its reader without waiting for acknowledgement", { timeout: 2000 }, async () => {
+ let cancelled = false;
+ const fixture = tarballProof();
+ fixture.options.fetchMetadata = async (url) => ({ status: 200, redirected: false, url,
+ body: new Response(new ReadableStream({
+ cancel() { cancelled = true; return new Promise(() => {}); },
+ })).body });
+ const budget = new AbortController();
+ const timer = setTimeout(() => budget.abort(new Error("fixture body budget expired")), 100);
+ const started = Date.now();
+ try {
+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {
+ ...fixture.options, signal: budget.signal,
+ }), /fixture body budget expired/);
+ assert.equal(cancelled, true);
+ assert.ok(Date.now() - started < 1000);
+ } finally { clearTimeout(timer); }
+});
+
+test("metadata headers cannot outlive the budget and late response bodies are cancelled", { timeout: 2000 }, async () => {
+ let headersArrive;
+ let cancelled = false;
+ const fixture = tarballProof();
+ fixture.options.fetchMetadata = (url) => new Promise((resolve) => {
+ headersArrive = () => resolve({ status: 200, redirected: false, url,
+ body: new Response(new ReadableStream({ cancel() { cancelled = true; } })).body });
+ });
+ const budget = new AbortController();
+ const timer = setTimeout(() => budget.abort(new Error("fixture headers budget expired")), 100);
+ const started = Date.now();
+ try {
+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {
+ ...fixture.options, signal: budget.signal,
+ }), /fixture headers budget expired/);
+ assert.ok(Date.now() - started < 1000);
+ headersArrive();
+ await new Promise((resolve) => setImmediate(resolve));
+ assert.equal(cancelled, true);
+ } finally { clearTimeout(timer); }
+});
+
+test("real npm effective registry reads isolated prefix, user, and environment configuration", async () => {
+ const root = mkdtempSync(path.join(os.tmpdir(), "canary-config-proof-"));
+ mkdirSync(path.join(root, "prefix"));
+ writeFileSync(path.join(root, "global.npmrc"), "");
+ const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_")));
+ env.npm_config_userconfig = path.join(root, "user.npmrc");
+ env.npm_config_globalconfig = path.join(root, "global.npmrc");
+ try {
+ for (const [user, project, override, expected] of [
+ ["registry=https://registry.npmjs.org/\n", "", undefined, true],
+ ["registry=https://registry.example.invalid/\n", "@paperclipai:registry=https://registry.npmjs.org/\n", undefined, true],
+ ["registry=https://registry.npmjs.org/\n", "@paperclipai:registry=https://registry.example.invalid/\n", undefined, false],
+ ["registry=https://registry.npmjs.org/\n", "", "https://registry.example.invalid/", false],
+ ]) {
+ writeFileSync(path.join(root, "user.npmrc"), user);
+ writeFileSync(path.join(root, "prefix", ".npmrc"), project);
+ const fixture = tarballProof();
+ delete fixture.options.runConfig;
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, {
+ ...fixture.options, prefix: path.join(root, "prefix"), env: { ...env, ...(override ? { npm_config_registry: override } : {}) },
+ }), expected);
+ assert.equal(fixture.calls.length, expected ? 1 : 0);
+ }
+ } finally { rmSync(root, { recursive: true, force: true }); }
+});
+
+async function stalledMetadata(run) {
+ let closed = false;
+ let closing = false;
+ let observeClose;
+ const connectionClosed = new Promise((resolve) => { observeClose = resolve; });
+ const server = createServer((_request, response) => {
+ response.on("close", () => { closed = true; observeClose(); });
+ response.writeHead(200, { "content-type": "application/json" });
+ response.write('{"name":');
+ });
+ // An HTTP client can replace an aborted pooled connection while shutdown
+ // starts. Own late connections too, so server.close cannot wait on them.
+ server.on("connection", (socket) => { if (closing) socket.destroy(); });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const fixture = tarballProof();
+ fixture.options.fetchMetadata = async (url, options) => {
+ const response = await fetch(`http://127.0.0.1:${server.address().port}/metadata`, options);
+ Object.defineProperty(response, "url", { value: url });
+ return response;
+ };
+ let closeDeadline;
+ try {
+ await run(fixture.options);
+ await Promise.race([connectionClosed, new Promise((_, reject) => {
+ closeDeadline = setTimeout(() => reject(new Error("metadata connection outlived cancellation")), 1000);
+ })]);
+ assert.equal(closed, true, "metadata connection must close before fixture-forced cleanup");
+ }
+ finally {
+ clearTimeout(closeDeadline);
+ closing = true;
+ await new Promise((resolve) => {
+ server.close(resolve);
+ server.closeAllConnections();
+ });
+ }
+}
+
+test("shared acquisition cancellation aborts the metadata body before the lookup deadline", async () => {
+ await stalledMetadata(async (options) => {
+ const budget = new AbortController();
+ const timer = setTimeout(() => budget.abort(new Error("fixture shared budget expired")), 250);
+ try {
+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { ...options, signal: budget.signal }), /fixture shared budget expired/);
+ } finally { clearTimeout(timer); }
+ });
+});
+
+test("the short config and metadata deadline also covers a stalled real HTTP body", { timeout: 10_000 }, async () => {
+ assert.equal(TARBALL_METADATA_TIMEOUT_MS, 5000);
+ const started = Date.now();
+ await stalledMetadata(async (options) => {
+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, options), false);
+ });
+ assert.ok(Date.now() - started < 7000, "metadata must not consume the 120s acquisition budget");
+});
+
+test("a verified canary tarball404 retries acquisition and runs onboarding once", async () => {
+ const result = await canaryStartup({ mode: "tarball-recover" });
+ assert.equal(result.code, 0, result.output);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);
+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);
+ assert.doesNotMatch(result.output, /fixture config diagnostic/);
+});
+
+test("persistent promised tarball404s stop at three installs without a final lookup or onboarding", async () => {
+ const result = await canaryStartup({ mode: "tarball-permanent" });
+ assert.equal(result.code, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 3);
+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 2);
+ assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 2);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
+ assert.match(result.output, /installation failed.*onboarding was not started/);
+});
+
+test("private or oversized config and unpublished metadata404 fail without retry or leaking config", async () => {
+ for (const input of [{ config: "private" }, { config: "large" }, { metadata: "missing" }]) {
+ const result = await canaryStartup({ mode: "tarball-permanent", ...input });
+ assert.equal(result.code, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
+ assert.doesNotMatch(result.output, /fixture-user|fixture-password|fixture config diagnostic/);
+ }
+});
+
+test("truncated npm output cannot hide an earlier auth failure and qualify a tarball retry", async () => {
+ const result = await canaryStartup({ mode: "tarball-large" });
+ assert.equal(result.code, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
+});
+
+test("tarball recovery does not replay onboarding failures or change ETARGET for other exact versions", async () => {
+ const failed = await canaryStartup({ mode: "tarball-recover", onboard: "fail" });
+ assert.equal(failed.code, 1);
+ assert.equal(failed.calls.filter((call) => call.kind === "npm").length, 2);
+ assert.equal(failed.calls.filter((call) => call.kind === "onboard").length, 1);
+ assert.match(failed.output, /onboarding failed \(exit 17\)/);
+ const other = await canaryStartup({ mode: "recover", version: "2026.1009.0-nightly.1" });
+ assert.equal(other.code, 0, other.output);
+ assert.equal(other.calls.filter((call) => call.kind === "npm").length, 2);
+ assert.equal(other.calls.filter((call) => call.kind === "onboard").length, 1);
+ assert.equal(other.calls.filter((call) => call.kind === "config").length, 0);
+});
+
+test("existing ETARGET character-tail behavior survives long multibyte npm diagnostics", async () => {
+ const result = await canaryStartup({ mode: "multibyte-recover" });
+ assert.equal(result.code, 0, result.output);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);
+});
+
+test("metadata lookup and backoff both consume the unchanged shared acquisition budget", async () => {
+ for (const input of [{ metadata: "hang" }, { delay: 5000 }]) {
+ const result = await canaryStartup({ mode: "tarball-permanent", budget: 1000, ...input });
+ assert.equal(result.code, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
+ assert.match(result.output, /exceeded its startup budget/);
+ assert.ok(result.elapsed < 4000, `shared-budget cancellation took ${result.elapsed}ms`);
+ }
+});
+
+test("cancellation reaps a config process and its descendant that ignore SIGTERM", async () => {
+ const result = await canaryStartup({ mode: "tarball-permanent", config: "hang", cancelAtKind: "config-descendant" });
+ assert.equal(result.code, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
+ for (const child of result.calls.filter((call) => ["config", "config-descendant"].includes(call.kind))) {
+ assert.throws(() => process.kill(child.pid, 0), { code: "ESRCH" });
+ }
+ assert.match(result.output, /stopped by SIGTERM/);
+});
diff --git a/tests/canary-onboarding/start-published-canary.mjs b/tests/canary-onboarding/start-published-canary.mjs
index d52d0f87da..f294d8a9f1 100644
--- a/tests/canary-onboarding/start-published-canary.mjs
+++ b/tests/canary-onboarding/start-published-canary.mjs
@@ -2,21 +2,148 @@ import { spawn } from "node:child_process";
import { mkdtemp, rm } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
+import { loadReleaseManifest } from "../../scripts/release-package-map.mjs";
// Leave three minutes for onboarding inside Playwright's existing five-minute
// startup deadline. This budget includes every install attempt and backoff.
export const INSTALL_BUDGET_MS = 120_000;
const MAX_INSTALL_ATTEMPTS = 3;
const STDERR_TAIL_CHARS = 64 * 1024;
+export const TARBALL_METADATA_TIMEOUT_MS = 5_000;
+const METADATA_LIMIT_BYTES = 64 * 1024;
+const PRIMARY_REGISTRY = "https://registry.npmjs.org/";
+const RELEASE_PACKAGE_NAMES = new Set(loadReleaseManifest()
+ .filter((entry) => entry.publishFromCi === true).map((entry) => entry.name));
-function run(command, args, { env, signal, captureStderr = false }) {
+function withAbort(pending, signal) {
+ return new Promise((resolve, reject) => {
+ const abort = () => {
+ signal.removeEventListener("abort", abort);
+ reject(signal.reason);
+ };
+ const finish = (settle) => (value) => {
+ signal.removeEventListener("abort", abort);
+ settle(value);
+ };
+ // Keep handlers on the original operation even after the bounded wait
+ // ends. A transport need not settle immediately when its signal aborts.
+ Promise.resolve(pending).then(finish(resolve), finish(reject));
+ signal.addEventListener("abort", abort, { once: true });
+ if (signal.aborted) abort();
+ });
+}
+
+export async function canRetryPublishedTarball404(stderr, version, {
+ signal, prefix, env = process.env, fetchMetadata = globalThis.fetch, runConfig = run,
+}) {
+ // A package metadata 404, a third-party artifact, or another release must not
+ // be mistaken for availability of this canary's announced public tarball.
+ if (!/^\d+\.\d+\.\d+-canary\.\d+$/.test(version)) return false;
+ const codes = [...stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)];
+ const final = codes.at(-1);
+ if (codes.length !== 1 || final?.[1] !== "E404") return false;
+ const block = stderr.slice(final.index);
+ const requests = [...block.matchAll(/^npm (?:error|ERR!) 404 Not Found - GET (\S+) - Not found\r?$/gm)];
+ const httpLines = stderr.match(/^npm (?:error|ERR!).* - (?:GET|HEAD|POST|PUT|DELETE|PATCH) /gm) ?? [];
+ if (requests.length !== 1 || httpLines.length !== 1) return false;
+ const tarball = requests[0][1];
+ const packageName = [...RELEASE_PACKAGE_NAMES].find((name) => {
+ const basename = name.split("/").at(-1);
+ return tarball === `${PRIMARY_REGISTRY}${name}/-/${basename}-${version}.tgz`;
+ });
+ if (!packageName) return false;
+ const resources = [...block.matchAll(/^npm (?:error|ERR!) 404\s+(?:The requested resource )?'([^']+)'(?: could not be found| is not in this registry)/gm)];
+ if (resources.some((resource) => resource[1] !== `${packageName}@${tarball}`)) return false;
+ signal.throwIfAborted();
+ const timeout = new AbortController();
+ const timer = setTimeout(() => timeout.abort(new Error("Canary tarball metadata lookup timed out")), TARBALL_METADATA_TIMEOUT_MS);
+ const lookupSignal = AbortSignal.any([signal, timeout.signal]);
+ const metadataUrl = `${PRIMARY_REGISTRY}${encodeURIComponent(packageName)}/${version}`;
+ let response;
+ try {
+ // Query only registry values in the failed install's effective prefix,
+ // inherited cwd and environment. Keep even warnings out of the logs:
+ // configuration values can contain credentials. Config and HTTP share
+ // this short deadline, inside the unchanged installation budget.
+ const config = await runConfig(process.platform === "win32" ? "npm.cmd" : "npm", [
+ "config", "get", "registry", "@paperclipai:registry", "--prefix", prefix,
+ ], { env, signal: lookupSignal, captureStdout: true, captureStderr: true, quiet: true, outputLimit: 4096 });
+ if (config.code !== 0 || config.signal || config.outputTruncated) return false;
+ const lines = config.stdout.trim().split(/\r?\n/);
+ if (lines.length !== 2 || !lines[0].startsWith("registry=") || !lines[1].startsWith("@paperclipai:registry=")) return false;
+ const globalRegistry = lines[0].slice("registry=".length);
+ const scoped = lines[1].slice("@paperclipai:registry=".length);
+ const effectiveRegistry = packageName.startsWith("@") && scoped !== "undefined" ? scoped : globalRegistry;
+ if (![PRIMARY_REGISTRY, PRIMARY_REGISTRY.slice(0, -1)].includes(effectiveRegistry)) return false;
+ response = await withAbort(Promise.resolve(fetchMetadata(metadataUrl, {
+ signal: lookupSignal, redirect: "error", credentials: "omit",
+ headers: { accept: "application/json", "cache-control": "no-cache" },
+ })).then((arrived) => {
+ if (lookupSignal.aborted) void arrived.body?.cancel().catch(() => {});
+ return arrived;
+ }), lookupSignal);
+ if (response.status !== 200 || response.redirected || response.url !== metadataUrl || !response.body) return false;
+ const reader = response.body.getReader();
+ const cancelReader = () => { void reader.cancel().catch(() => {}); };
+ lookupSignal.addEventListener("abort", cancelReader, { once: true });
+ const chunks = [];
+ let bytes = 0;
+ try {
+ while (true) {
+ lookupSignal.throwIfAborted();
+ const { done, value } = await withAbort(reader.read(), lookupSignal);
+ if (done) break;
+ bytes += value.byteLength;
+ if (bytes > METADATA_LIMIT_BYTES) return false;
+ chunks.push(value);
+ }
+ } finally {
+ // Initiate cleanup without extending either deadline for a transport's
+ // pending/rejected cancellation acknowledgement.
+ lookupSignal.removeEventListener("abort", cancelReader);
+ cancelReader();
+ reader.releaseLock();
+ }
+ lookupSignal.throwIfAborted();
+ const manifest = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+ const integrity = manifest.dist?.integrity;
+ // This validates the integrity announced by primary metadata. No tarball
+ // is downloaded here, and this is not an independent content digest proof.
+ return manifest.name === packageName && manifest.version === version
+ && manifest.dist?.tarball === tarball && typeof integrity === "string"
+ && /^sha512-[A-Za-z0-9+/]{86}==$/.test(integrity)
+ && Buffer.from(integrity.slice(7), "base64").toString("base64") === integrity.slice(7);
+ } catch {
+ signal.throwIfAborted();
+ return false;
+ } finally {
+ void response?.body?.cancel().catch(() => {});
+ clearTimeout(timer);
+ }
+}
+
+function run(command, args, { env, signal, captureStderr = false, captureStdout = false, quiet = false, outputLimit = STDERR_TAIL_CHARS }) {
return new Promise((resolve, reject) => {
signal.throwIfAborted();
const grouped = process.platform !== "win32";
const child = spawn(command, args, {
- env, detached: grouped, stdio: ["ignore", "inherit", captureStderr ? "pipe" : "inherit"],
+ env, detached: grouped, stdio: ["ignore", captureStdout ? "pipe" : "inherit", captureStderr ? "pipe" : "inherit"],
});
- let stderr = "";
+ let stderr = quiet ? Buffer.alloc(0) : "";
+ let stdout = quiet ? Buffer.alloc(0) : "";
+ let outputTruncated = false;
+ const retain = (previous, chunk) => {
+ if (quiet) {
+ outputTruncated ||= previous.length + chunk.byteLength > outputLimit;
+ return Buffer.concat([previous, chunk]).subarray(-outputLimit);
+ }
+ // Preserve the existing install stderr character tail and ETARGET
+ // behavior, including multibyte diagnostics. Only private config capture
+ // uses a byte bound; any truncated E404 evidence is rejected.
+ const next = previous + chunk.toString();
+ outputTruncated ||= next.length > outputLimit;
+ return next.slice(-outputLimit);
+ };
let stopping;
const kill = (kind) => {
try {
@@ -34,15 +161,20 @@ function run(command, args, { env, signal, captureStderr = false }) {
};
signal.addEventListener("abort", stop, { once: true });
child.stderr?.on("data", (chunk) => {
- process.stderr.write(chunk);
- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARS);
+ if (!quiet) process.stderr.write(chunk);
+ stderr = retain(stderr, chunk);
+ });
+ child.stdout?.on("data", (chunk) => {
+ if (!quiet) process.stdout.write(chunk);
+ stdout = retain(stdout, chunk);
});
child.on("error", reject);
child.on("close", async (code, exitSignal) => {
signal.removeEventListener("abort", stop);
await stopping;
if (signal.aborted) reject(signal.reason);
- else resolve({ code, signal: exitSignal, stderr });
+ else resolve({ code, signal: exitSignal,
+ stderr: quiet ? stderr.toString("utf8") : stderr, stdout: quiet ? stdout.toString("utf8") : stdout, outputTruncated });
});
});
}
@@ -85,7 +217,10 @@ export async function startPublishedCanary({
if (result.code === 0) break;
const codes = [...result.stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)];
const missingVersion = codes.at(-1)?.[1] === "ETARGET";
- if (!missingVersion || result.signal || attempt === MAX_INSTALL_ATTEMPTS) {
+ const retryable = !result.signal && attempt < MAX_INSTALL_ATTEMPTS
+ && (missingVersion || (!result.outputTruncated
+ && await canRetryPublishedTarball404(result.stderr, version, { signal: installSignal, prefix, env })));
+ if (!retryable) {
throw new Error(`Canary npm installation failed (exit ${result.code ?? result.signal}); onboarding was not started`);
}
await rm(prefix, { recursive: true, force: true });