The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(release): retry verified canary tarball 404s (#15768)
Devin Foley4f680bc85c82
diff --git a/doc/RELEASING.md b/doc/RELEASING.mdindex 771d3acb5b..dbf40956a5 100644--- a/doc/RELEASING.md+++ b/doc/RELEASING.md@@ -104,7 +104,7 @@ It:- publishes the user-facing `paperclipai` package last, so `paperclipai@canary` does not advance before the full package set exists- verifies that `canary` resolves to the just-published version and that published internal dependencies exist on npm- installs `paperclipai@canary` into a clean temporary prefix as the final npm gate-- starts the exact published canary through a separate fresh npm install for the onboarding smoke; only npm `ETARGET` install failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline+- starts the exact published canary through a separate fresh npm install for the onboarding smoke; npm `ETARGET` and verified canary tarball availability failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline- fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional- creates a git tag `canary/vYYYY.MDD.P-canary.N`@@ -116,6 +116,23 @@ current patches and manifests while the separate lockfile-refresh PR is stillpending, without resolving new dependencies in the smoke job. The smoke still installs and teststhe exact published canary version, not a workspace build.+The tarball retry applies only to an unambiguous npm `E404` for a canonical+public-registry tarball of a CI-published Paperclip package at that exact canary+version. The helper queries npm's effective global and applicable scoped registry+in the failed install's prefix, working directory, and environment. A custom or+unknown effective registry fails. Configuration output stays private in memory.+Fresh primary-registry metadata must match the package, version, and tarball URL+and announce a valid SHA512 integrity value. This establishes the announced+artifact identity; it does not download the tarball or independently verify its+contents. The actual npm install still checks tarball integrity.++Configuration and metadata checks share a five-second deadline, including the+response body, within the same two-minute acquisition budget. Metadata is limited+to 64 KiB. Redirects, authentication errors, missing metadata, conflicting npm+errors, and other artifacts fail. A persistent promised-tarball failure also+fails after three installs. Onboarding errors never restart acquisition or+repeat onboarding.+The lockfile artifact is retained for 14 days. A publisher-job rerun replaces itssource-named artifact; a smoke-only rerun uses the existing artifact. A missingartifact fails the job rather than falling back to a different dependency set.diff --git a/scripts/__tests__/canary-startup-fixture.mjs b/scripts/__tests__/canary-startup-fixture.mjsindex d75d1fa944..4a51ce621b 100644--- a/scripts/__tests__/canary-startup-fixture.mjs+++ b/scripts/__tests__/canary-startup-fixture.mjs@@ -10,7 +10,8 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."// Exercise the real startup helper and child-process boundary without registry// traffic. The npm fixture installs a real executable into its isolated prefix.-export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false }) {+export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false,+ config = "public", metadata = "valid", version = "2026.1009.0-canary.1" }) {const root = mkdtempSync(path.join(os.tmpdir(), "canary-startup-test-"));const calls = path.join(root, "calls.jsonl");const fixtureBin = path.join(root, "bin");@@ -39,15 +40,35 @@ if (process.env.FIXTURE_ONBOARD === "serve") {const fs = require("node:fs");const path = require("node:path");const args = process.argv.slice(2);+if (args[0] === "config") {+ fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config", pid:process.pid, args}) + "\\n");+ if (process.env.FIXTURE_CONFIG === "hang") {+ const { spawn } = require("node:child_process");+ const child = spawn(process.execPath, ["-e", "process.on('SIGTERM',()=>{}); setTimeout(()=>process.exit(0),10000)"], {stdio:"ignore"});+ fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config-descendant", pid:child.pid}) + "\\n");+ process.on("SIGTERM", () => {});+ setTimeout(() => process.exit(0), 10000);+ } else {+ if (process.env.FIXTURE_CONFIG === "large") console.log("x".repeat(8192));+ console.log("registry=" + (process.env.FIXTURE_CONFIG === "private" ? "https://fixture-user:[email protected]/" : "https://registry.npmjs.org/") + "\\n@paperclipai:registry=undefined");+ console.error("fixture config diagnostic must stay private");+ }+} else {const entries = fs.existsSync(process.env.FIXTURE_CALLS) ? fs.readFileSync(process.env.FIXTURE_CALLS, "utf8").trim().split("\\n").filter(Boolean).map(JSON.parse) : [];const attempt = entries.filter(e => e.kind === "npm").length + 1;fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"npm", pid:process.pid, args, cache:process.env.npm_config_cache}) + "\\n");const mode = process.env.FIXTURE_MODE;if (mode === "hang") { setTimeout(() => {}, 10000); }else if (mode === "permanent" || (mode === "recover" && attempt === 1)) { console.error("npm error code ETARGET"); process.exit(1); }+else if (mode === "multibyte-recover" && attempt === 1) { console.error("npm error code ETARGET\\n" + "界".repeat(30 * 1024)); process.exit(1); }else if (mode === "auth") { console.error("npm error code E401"); process.exit(1); }else if (mode === "mixed") { console.error("npm error code ETARGET\\nnpm error code E401"); process.exit(1); }else if (mode === "network") { console.error("npm error code ECONNRESET"); process.exit(1); }+else if (mode === "tarball-permanent" || mode === "tarball-large" || (mode === "tarball-recover" && attempt === 1)) {+ const url = "https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz";+ if (mode === "tarball-large") console.error("npm error code E401\\n" + "x".repeat(70 * 1024));+ console.error("npm error code E404\\nnpm error 404 Not Found - GET " + url + " - Not found"); process.exit(1);+}else {const prefix = args[args.indexOf("--prefix") + 1];const bin = path.join(prefix, "node_modules", ".bin");@@ -55,12 +76,24 @@ else {fs.writeFileSync(path.join(bin, "paperclipai"), ${JSON.stringify(cli).replaceAll('$','\$')});fs.chmodSync(path.join(bin, "paperclipai"), 0o755);}+}`);chmodSync(npmFixture, 0o755);const helper = new URL("../../tests/canary-onboarding/start-published-canary.mjs", import.meta.url).href;const driver = path.join(root, "driver.mjs");writeFileSync(driver, `import { startPublishedCanary } from ${JSON.stringify(helper)};-try { await startPublishedCanary({version:"2026.1009.0-canary.1", workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); }+import { appendFileSync } from "node:fs";+globalThis.fetch = async (url, options) => {+ appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"metadata", url, redirect:options.redirect, credentials:options.credentials, headers:options.headers}) + "\\n");+ const manifest = {name:"@paperclipai/server",version:process.env.PAPERCLIPAI_VERSION,dist:{tarball:"https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz",integrity:"sha512-" + Buffer.alloc(64).toString("base64")}};+ const body = process.env.FIXTURE_METADATA === "hang" ? new ReadableStream({+ start(controller) { options.signal.addEventListener("abort", () => controller.error(options.signal.reason), {once:true}); },+ }) : JSON.stringify(manifest);+ const response = new Response(body, {status:process.env.FIXTURE_METADATA === "missing" ? 404 : 200});+ Object.defineProperty(response,"url",{value:url});+ return response;+};+try { await startPublishedCanary({version:${JSON.stringify(version)}, workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); }catch (error) { console.error(error.message); process.exitCode = 1; }`);const playwrightConfig = path.join(root, "playwright.config.mjs");@@ -80,8 +113,8 @@ export default {...original, testDir:${JSON.stringify(root)}, testMatch:"fixture? [fileURLToPath(helper), "2026.1009.0-canary.1", root, path.join(root, "data")]: [driver], { cwd: repoRoot, env: {...process.env, PATH: fixtureBin + path.delimiter + process.env.PATH,- FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port),- PAPERCLIPAI_VERSION: "2026.1009.0-canary.1", PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"),+ FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port), FIXTURE_CONFIG: config, FIXTURE_METADATA: metadata,+ PAPERCLIPAI_VERSION: version, PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"),npm_config_cache: path.join(root, "fresh-cache"),}, stdio: ["ignore", "pipe", "pipe"] });let output = "";diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjsindex 9e5364c869..043636edb9 100644--- a/scripts/__tests__/release-verify-workflow.test.mjs+++ b/scripts/__tests__/release-verify-workflow.test.mjs@@ -5,7 +5,9 @@ import os from "node:os";import path from "node:path";import { fileURLToPath } from "node:url";import test from "node:test";+import { createServer } from "node:http";import { canaryStartup } from "./canary-startup-fixture.mjs";+import { canRetryPublishedTarball404, TARBALL_METADATA_TIMEOUT_MS } from "../../tests/canary-onboarding/start-published-canary.mjs";const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)),@@ -830,3 +832,327 @@ test("nested ETARGET output does not retry a final authentication or network faiassert.equal(result.calls.length, 1);}});++const exactCanary = "2026.1009.0-canary.1";+const canaryTarball = `https://registry.npmjs.org/@paperclipai/server/-/server-${exactCanary}.tgz`;+const tarballFailure = (url = canaryTarball, prefix = "npm error") => `${prefix} code E404\n${prefix} 404 Not Found - GET ${url} - Not found\n${prefix} 404\n${prefix} 404 The requested resource '@paperclipai/server@${url}' could not be found or you do not have permission to access it.\n`;++function tarballProof({ config = {}, manifest = {}, status = 200, redirected = false, responseUrl, rawBody } = {}) {+ const calls = [];+ const dist = { tarball: canaryTarball, integrity: `sha512-${Buffer.alloc(64).toString("base64")}`, ...manifest.dist };+ const data = { name: "@paperclipai/server", version: exactCanary, ...manifest, dist };+ return { calls, options: {+ signal: new AbortController().signal, prefix: "/fixture/install", env: {},+ runConfig: async (command, args, options) => {+ calls.push({ kind: "config", command, args, options });+ return { code: 0, stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=undefined\n", ...config };+ },+ fetchMetadata: async (url, options) => {+ calls.push({ kind: "metadata", url, options });+ return { status, redirected, url: responseUrl ?? url, body: new Response(rawBody ?? JSON.stringify(data)).body };+ },+ } };+}++test("exact published tarball proof uses quiet effective config and fresh public metadata without credentials", async () => {+ for (const prefix of ["npm error", "npm ERR!"]) {+ const fixture = tarballProof();+ assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball, prefix), exactCanary, fixture.options), true);+ const [config, metadata] = fixture.calls;+ assert.deepEqual(config.args, ["config", "get", "registry", "@paperclipai:registry", "--prefix", "/fixture/install"]);+ assert.equal(config.options.env, fixture.options.env);+ assert.equal(config.options.quiet, true);+ assert.equal(config.options.captureStdout, true);+ assert.equal(config.options.captureStderr, true);+ assert.equal(config.options.outputLimit, 4096);+ assert.equal(metadata.url, `https://registry.npmjs.org/%40paperclipai%2Fserver/${exactCanary}`);+ assert.equal(metadata.options.signal, config.options.signal);+ assert.equal(metadata.options.redirect, "error");+ assert.equal(metadata.options.credentials, "omit");+ assert.deepEqual(metadata.options.headers, { accept: "application/json", "cache-control": "no-cache" });+ }+});++test("mixed npm errors and noncanonical or unreleased tarballs never read config or metadata", async () => {+ const errors = [+ "npm error code ETARGET\n", "npm error code E401\n", "npm error code ECONNRESET\n",+ `${tarballFailure()}npm error code E401\n`, `${tarballFailure()}npm error code 1\n`,+ `npm error code ENEEDAUTH\n${tarballFailure()}`, `${tarballFailure()}${tarballFailure()}`,+ `${tarballFailure()}npm error 404 Not Found - GET https://registry.example.invalid/a.tgz - Not found\n`,+ `${tarballFailure()}npm error 404 The requested resource 'other@${canaryTarball}' could not be found\n`,+ tarballFailure().replace(" - GET ", " - PUT "),+ ...[+ canaryTarball + "?token=fixture", canaryTarball + "#fragment", canaryTarball.replace("npmjs.org/", "npmjs.org:443/"),+ canaryTarball.replace("npmjs.org/", "npmjs.org.example.invalid/"), canaryTarball.replace("npmjs.org/", "[email protected]/"),+ canaryTarball.replace("https:", "http:"), canaryTarball.replace("@paperclipai/server", "%40paperclipai%2Fserver"),+ canaryTarball.replace("@paperclipai/server", "@paperclipai/../server"), canaryTarball.replace(exactCanary, "2026.1009.0-canary.2"),+ "https://registry.npmjs.org/@paperclipai/not-released/-/not-released-2026.1009.0-canary.1.tgz",+ "https://registry.npmjs.org/@paperclipai/plugin-createos/-/plugin-createos-2026.1009.0-canary.1.tgz",+ "https://registry.npmjs.org/%40paperclipai%2Fserver/2026.1009.0-canary.1",+ ].map((url) => tarballFailure(url)),+ ];+ for (const error of errors) {+ const fixture = tarballProof();+ assert.equal(await canRetryPublishedTarball404(error, exactCanary, fixture.options), false, error);+ assert.deepEqual(fixture.calls, [], error);+ }+ for (const version of ["2026.1009.0", "2026.1009.0-nightly.1", "2026.1009.0-beta.1", "2026.1009.0-canary.1+build"]) {+ const fixture = tarballProof();+ assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball.replace(exactCanary, version)), version, fixture.options), false);+ assert.deepEqual(fixture.calls, []);+ }+});++test("effective registry authority respects scope precedence and rejects unknown or private config", async () => {+ const configs = [+ [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],+ [{ stdout: "registry=undefined\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],+ [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=https://registry.example.invalid/\n" }, false],+ [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=undefined\n" }, false],+ [{ stdout: "registry=undefined\n@paperclipai:registry=undefined\n" }, false],+ [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=\n" }, false],+ [{ stdout: "registry=https://fixture:[email protected]/\n@paperclipai:registry=undefined\n" }, false],+ [{ stdout: "unexpected output\n" }, false], [{ code: 1 }, false], [{ signal: "SIGTERM" }, false], [{ outputTruncated: true }, false],+ ];+ for (const [config, expected] of configs) {+ const fixture = tarballProof({ config });+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), expected);+ assert.equal(fixture.calls.filter((call) => call.kind === "metadata").length, expected ? 1 : 0);+ }+});++test("metadata must identify the exact published artifact with valid SHA512 syntax, not merely exist", async () => {+ const fixtures = [+ { status: 404 }, { status: 401 }, { status: 403 }, { status: 302 }, { redirected: true },+ { responseUrl: "https://registry.example.invalid/metadata" }, { rawBody: "not json" },+ { manifest: { name: "@paperclipai/shared" } }, { manifest: { version: "2026.1009.0-canary.2" } },+ { manifest: { dist: { tarball: canaryTarball + "?query=1" } } },+ ...[undefined, "", "sha512-A===", `sha256-${Buffer.alloc(32).toString("base64")}`, `sha512-${Buffer.alloc(64).toString("base64")} extra`]+ .map((integrity) => ({ manifest: { dist: { integrity } } })),+ { rawBody: JSON.stringify({ padding: "x".repeat(65 * 1024) }) },+ ];+ for (const input of fixtures) {+ const fixture = tarballProof(input);+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false, JSON.stringify(input).slice(0, 200));+ }+});++test("rejected and oversized native response bodies cancel without waiting for slow or rejected cleanup", async () => {+ for (const [status, pending] of [[200, true], [200, false], [404, true], [404, false]]) {+ let cancelled = false;+ const fixture = tarballProof();+ fixture.options.fetchMetadata = async (url) => ({ status, redirected: false, url, body: new Response(new ReadableStream({+ start(controller) { controller.enqueue(new Uint8Array(65 * 1024)); },+ cancel() { cancelled = true; return pending ? new Promise(() => {}) : Promise.reject(new Error("fixture rejected cancellation")); },+ })).body });+ const started = Date.now();+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false);+ assert.equal(cancelled, true);+ assert.ok(Date.now() - started < 1000, "cancellation acknowledgement must not extend a rejection");+ }+});++test("aborting a stalled metadata read cancels its reader without waiting for acknowledgement", { timeout: 2000 }, async () => {+ let cancelled = false;+ const fixture = tarballProof();+ fixture.options.fetchMetadata = async (url) => ({ status: 200, redirected: false, url,+ body: new Response(new ReadableStream({+ cancel() { cancelled = true; return new Promise(() => {}); },+ })).body });+ const budget = new AbortController();+ const timer = setTimeout(() => budget.abort(new Error("fixture body budget expired")), 100);+ const started = Date.now();+ try {+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {+ ...fixture.options, signal: budget.signal,+ }), /fixture body budget expired/);+ assert.equal(cancelled, true);+ assert.ok(Date.now() - started < 1000);+ } finally { clearTimeout(timer); }+});++test("metadata headers cannot outlive the budget and late response bodies are cancelled", { timeout: 2000 }, async () => {+ let headersArrive;+ let cancelled = false;+ const fixture = tarballProof();+ fixture.options.fetchMetadata = (url) => new Promise((resolve) => {+ headersArrive = () => resolve({ status: 200, redirected: false, url,+ body: new Response(new ReadableStream({ cancel() { cancelled = true; } })).body });+ });+ const budget = new AbortController();+ const timer = setTimeout(() => budget.abort(new Error("fixture headers budget expired")), 100);+ const started = Date.now();+ try {+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {+ ...fixture.options, signal: budget.signal,+ }), /fixture headers budget expired/);+ assert.ok(Date.now() - started < 1000);+ headersArrive();+ await new Promise((resolve) => setImmediate(resolve));+ assert.equal(cancelled, true);+ } finally { clearTimeout(timer); }+});++test("real npm effective registry reads isolated prefix, user, and environment configuration", async () => {+ const root = mkdtempSync(path.join(os.tmpdir(), "canary-config-proof-"));+ mkdirSync(path.join(root, "prefix"));+ writeFileSync(path.join(root, "global.npmrc"), "");+ const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_")));+ env.npm_config_userconfig = path.join(root, "user.npmrc");+ env.npm_config_globalconfig = path.join(root, "global.npmrc");+ try {+ for (const [user, project, override, expected] of [+ ["registry=https://registry.npmjs.org/\n", "", undefined, true],+ ["registry=https://registry.example.invalid/\n", "@paperclipai:registry=https://registry.npmjs.org/\n", undefined, true],+ ["registry=https://registry.npmjs.org/\n", "@paperclipai:registry=https://registry.example.invalid/\n", undefined, false],+ ["registry=https://registry.npmjs.org/\n", "", "https://registry.example.invalid/", false],+ ]) {+ writeFileSync(path.join(root, "user.npmrc"), user);+ writeFileSync(path.join(root, "prefix", ".npmrc"), project);+ const fixture = tarballProof();+ delete fixture.options.runConfig;+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, {+ ...fixture.options, prefix: path.join(root, "prefix"), env: { ...env, ...(override ? { npm_config_registry: override } : {}) },+ }), expected);+ assert.equal(fixture.calls.length, expected ? 1 : 0);+ }+ } finally { rmSync(root, { recursive: true, force: true }); }+});++async function stalledMetadata(run) {+ let closed = false;+ let closing = false;+ let observeClose;+ const connectionClosed = new Promise((resolve) => { observeClose = resolve; });+ const server = createServer((_request, response) => {+ response.on("close", () => { closed = true; observeClose(); });+ response.writeHead(200, { "content-type": "application/json" });+ response.write('{"name":');+ });+ // An HTTP client can replace an aborted pooled connection while shutdown+ // starts. Own late connections too, so server.close cannot wait on them.+ server.on("connection", (socket) => { if (closing) socket.destroy(); });+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));+ const fixture = tarballProof();+ fixture.options.fetchMetadata = async (url, options) => {+ const response = await fetch(`http://127.0.0.1:${server.address().port}/metadata`, options);+ Object.defineProperty(response, "url", { value: url });+ return response;+ };+ let closeDeadline;+ try {+ await run(fixture.options);+ await Promise.race([connectionClosed, new Promise((_, reject) => {+ closeDeadline = setTimeout(() => reject(new Error("metadata connection outlived cancellation")), 1000);+ })]);+ assert.equal(closed, true, "metadata connection must close before fixture-forced cleanup");+ }+ finally {+ clearTimeout(closeDeadline);+ closing = true;+ await new Promise((resolve) => {+ server.close(resolve);+ server.closeAllConnections();+ });+ }+}++test("shared acquisition cancellation aborts the metadata body before the lookup deadline", async () => {+ await stalledMetadata(async (options) => {+ const budget = new AbortController();+ const timer = setTimeout(() => budget.abort(new Error("fixture shared budget expired")), 250);+ try {+ await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { ...options, signal: budget.signal }), /fixture shared budget expired/);+ } finally { clearTimeout(timer); }+ });+});++test("the short config and metadata deadline also covers a stalled real HTTP body", { timeout: 10_000 }, async () => {+ assert.equal(TARBALL_METADATA_TIMEOUT_MS, 5000);+ const started = Date.now();+ await stalledMetadata(async (options) => {+ assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, options), false);+ });+ assert.ok(Date.now() - started < 7000, "metadata must not consume the 120s acquisition budget");+});++test("a verified canary tarball404 retries acquisition and runs onboarding once", async () => {+ const result = await canaryStartup({ mode: "tarball-recover" });+ assert.equal(result.code, 0, result.output);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);+ assert.doesNotMatch(result.output, /fixture config diagnostic/);+});++test("persistent promised tarball404s stop at three installs without a final lookup or onboarding", async () => {+ const result = await canaryStartup({ mode: "tarball-permanent" });+ assert.equal(result.code, 1);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 3);+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 2);+ assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 2);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);+ assert.match(result.output, /installation failed.*onboarding was not started/);+});++test("private or oversized config and unpublished metadata404 fail without retry or leaking config", async () => {+ for (const input of [{ config: "private" }, { config: "large" }, { metadata: "missing" }]) {+ const result = await canaryStartup({ mode: "tarball-permanent", ...input });+ assert.equal(result.code, 1);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);+ assert.doesNotMatch(result.output, /fixture-user|fixture-password|fixture config diagnostic/);+ }+});++test("truncated npm output cannot hide an earlier auth failure and qualify a tarball retry", async () => {+ const result = await canaryStartup({ mode: "tarball-large" });+ assert.equal(result.code, 1);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);+});++test("tarball recovery does not replay onboarding failures or change ETARGET for other exact versions", async () => {+ const failed = await canaryStartup({ mode: "tarball-recover", onboard: "fail" });+ assert.equal(failed.code, 1);+ assert.equal(failed.calls.filter((call) => call.kind === "npm").length, 2);+ assert.equal(failed.calls.filter((call) => call.kind === "onboard").length, 1);+ assert.match(failed.output, /onboarding failed \(exit 17\)/);+ const other = await canaryStartup({ mode: "recover", version: "2026.1009.0-nightly.1" });+ assert.equal(other.code, 0, other.output);+ assert.equal(other.calls.filter((call) => call.kind === "npm").length, 2);+ assert.equal(other.calls.filter((call) => call.kind === "onboard").length, 1);+ assert.equal(other.calls.filter((call) => call.kind === "config").length, 0);+});++test("existing ETARGET character-tail behavior survives long multibyte npm diagnostics", async () => {+ const result = await canaryStartup({ mode: "multibyte-recover" });+ assert.equal(result.code, 0, result.output);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);+});++test("metadata lookup and backoff both consume the unchanged shared acquisition budget", async () => {+ for (const input of [{ metadata: "hang" }, { delay: 5000 }]) {+ const result = await canaryStartup({ mode: "tarball-permanent", budget: 1000, ...input });+ assert.equal(result.code, 1);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);+ assert.match(result.output, /exceeded its startup budget/);+ assert.ok(result.elapsed < 4000, `shared-budget cancellation took ${result.elapsed}ms`);+ }+});++test("cancellation reaps a config process and its descendant that ignore SIGTERM", async () => {+ const result = await canaryStartup({ mode: "tarball-permanent", config: "hang", cancelAtKind: "config-descendant" });+ assert.equal(result.code, 1);+ assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);+ assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);+ for (const child of result.calls.filter((call) => ["config", "config-descendant"].includes(call.kind))) {+ assert.throws(() => process.kill(child.pid, 0), { code: "ESRCH" });+ }+ assert.match(result.output, /stopped by SIGTERM/);+});diff --git a/tests/canary-onboarding/start-published-canary.mjs b/tests/canary-onboarding/start-published-canary.mjsindex d52d0f87da..f294d8a9f1 100644--- a/tests/canary-onboarding/start-published-canary.mjs+++ b/tests/canary-onboarding/start-published-canary.mjs@@ -2,21 +2,148 @@ import { spawn } from "node:child_process";import { mkdtemp, rm } from "node:fs/promises";import path from "node:path";import { fileURLToPath } from "node:url";+import { loadReleaseManifest } from "../../scripts/release-package-map.mjs";// Leave three minutes for onboarding inside Playwright's existing five-minute// startup deadline. This budget includes every install attempt and backoff.export const INSTALL_BUDGET_MS = 120_000;const MAX_INSTALL_ATTEMPTS = 3;const STDERR_TAIL_CHARS = 64 * 1024;+export const TARBALL_METADATA_TIMEOUT_MS = 5_000;+const METADATA_LIMIT_BYTES = 64 * 1024;+const PRIMARY_REGISTRY = "https://registry.npmjs.org/";+const RELEASE_PACKAGE_NAMES = new Set(loadReleaseManifest()+ .filter((entry) => entry.publishFromCi === true).map((entry) => entry.name));-function run(command, args, { env, signal, captureStderr = false }) {+function withAbort(pending, signal) {+ return new Promise((resolve, reject) => {+ const abort = () => {+ signal.removeEventListener("abort", abort);+ reject(signal.reason);+ };+ const finish = (settle) => (value) => {+ signal.removeEventListener("abort", abort);+ settle(value);+ };+ // Keep handlers on the original operation even after the bounded wait+ // ends. A transport need not settle immediately when its signal aborts.+ Promise.resolve(pending).then(finish(resolve), finish(reject));+ signal.addEventListener("abort", abort, { once: true });+ if (signal.aborted) abort();+ });+}++export async function canRetryPublishedTarball404(stderr, version, {+ signal, prefix, env = process.env, fetchMetadata = globalThis.fetch, runConfig = run,+}) {+ // A package metadata 404, a third-party artifact, or another release must not+ // be mistaken for availability of this canary's announced public tarball.+ if (!/^\d+\.\d+\.\d+-canary\.\d+$/.test(version)) return false;+ const codes = [...stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)];+ const final = codes.at(-1);+ if (codes.length !== 1 || final?.[1] !== "E404") return false;+ const block = stderr.slice(final.index);+ const requests = [...block.matchAll(/^npm (?:error|ERR!) 404 Not Found - GET (\S+) - Not found\r?$/gm)];+ const httpLines = stderr.match(/^npm (?:error|ERR!).* - (?:GET|HEAD|POST|PUT|DELETE|PATCH) /gm) ?? [];+ if (requests.length !== 1 || httpLines.length !== 1) return false;+ const tarball = requests[0][1];+ const packageName = [...RELEASE_PACKAGE_NAMES].find((name) => {+ const basename = name.split("/").at(-1);+ return tarball === `${PRIMARY_REGISTRY}${name}/-/${basename}-${version}.tgz`;+ });+ if (!packageName) return false;+ const resources = [...block.matchAll(/^npm (?:error|ERR!) 404\s+(?:The requested resource )?'([^']+)'(?: could not be found| is not in this registry)/gm)];+ if (resources.some((resource) => resource[1] !== `${packageName}@${tarball}`)) return false;+ signal.throwIfAborted();+ const timeout = new AbortController();+ const timer = setTimeout(() => timeout.abort(new Error("Canary tarball metadata lookup timed out")), TARBALL_METADATA_TIMEOUT_MS);+ const lookupSignal = AbortSignal.any([signal, timeout.signal]);+ const metadataUrl = `${PRIMARY_REGISTRY}${encodeURIComponent(packageName)}/${version}`;+ let response;+ try {+ // Query only registry values in the failed install's effective prefix,+ // inherited cwd and environment. Keep even warnings out of the logs:+ // configuration values can contain credentials. Config and HTTP share+ // this short deadline, inside the unchanged installation budget.+ const config = await runConfig(process.platform === "win32" ? "npm.cmd" : "npm", [+ "config", "get", "registry", "@paperclipai:registry", "--prefix", prefix,+ ], { env, signal: lookupSignal, captureStdout: true, captureStderr: true, quiet: true, outputLimit: 4096 });+ if (config.code !== 0 || config.signal || config.outputTruncated) return false;+ const lines = config.stdout.trim().split(/\r?\n/);+ if (lines.length !== 2 || !lines[0].startsWith("registry=") || !lines[1].startsWith("@paperclipai:registry=")) return false;+ const globalRegistry = lines[0].slice("registry=".length);+ const scoped = lines[1].slice("@paperclipai:registry=".length);+ const effectiveRegistry = packageName.startsWith("@") && scoped !== "undefined" ? scoped : globalRegistry;+ if (![PRIMARY_REGISTRY, PRIMARY_REGISTRY.slice(0, -1)].includes(effectiveRegistry)) return false;+ response = await withAbort(Promise.resolve(fetchMetadata(metadataUrl, {+ signal: lookupSignal, redirect: "error", credentials: "omit",+ headers: { accept: "application/json", "cache-control": "no-cache" },+ })).then((arrived) => {+ if (lookupSignal.aborted) void arrived.body?.cancel().catch(() => {});+ return arrived;+ }), lookupSignal);+ if (response.status !== 200 || response.redirected || response.url !== metadataUrl || !response.body) return false;+ const reader = response.body.getReader();+ const cancelReader = () => { void reader.cancel().catch(() => {}); };+ lookupSignal.addEventListener("abort", cancelReader, { once: true });+ const chunks = [];+ let bytes = 0;+ try {+ while (true) {+ lookupSignal.throwIfAborted();+ const { done, value } = await withAbort(reader.read(), lookupSignal);+ if (done) break;+ bytes += value.byteLength;+ if (bytes > METADATA_LIMIT_BYTES) return false;+ chunks.push(value);+ }+ } finally {+ // Initiate cleanup without extending either deadline for a transport's+ // pending/rejected cancellation acknowledgement.+ lookupSignal.removeEventListener("abort", cancelReader);+ cancelReader();+ reader.releaseLock();+ }+ lookupSignal.throwIfAborted();+ const manifest = JSON.parse(Buffer.concat(chunks).toString("utf8"));+ const integrity = manifest.dist?.integrity;+ // This validates the integrity announced by primary metadata. No tarball+ // is downloaded here, and this is not an independent content digest proof.+ return manifest.name === packageName && manifest.version === version+ && manifest.dist?.tarball === tarball && typeof integrity === "string"+ && /^sha512-[A-Za-z0-9+/]{86}==$/.test(integrity)+ && Buffer.from(integrity.slice(7), "base64").toString("base64") === integrity.slice(7);+ } catch {+ signal.throwIfAborted();+ return false;+ } finally {+ void response?.body?.cancel().catch(() => {});+ clearTimeout(timer);+ }+}++function run(command, args, { env, signal, captureStderr = false, captureStdout = false, quiet = false, outputLimit = STDERR_TAIL_CHARS }) {return new Promise((resolve, reject) => {signal.throwIfAborted();const grouped = process.platform !== "win32";const child = spawn(command, args, {- env, detached: grouped, stdio: ["ignore", "inherit", captureStderr ? "pipe" : "inherit"],+ env, detached: grouped, stdio: ["ignore", captureStdout ? "pipe" : "inherit", captureStderr ? "pipe" : "inherit"],});- let stderr = "";+ let stderr = quiet ? Buffer.alloc(0) : "";+ let stdout = quiet ? Buffer.alloc(0) : "";+ let outputTruncated = false;+ const retain = (previous, chunk) => {+ if (quiet) {+ outputTruncated ||= previous.length + chunk.byteLength > outputLimit;+ return Buffer.concat([previous, chunk]).subarray(-outputLimit);+ }+ // Preserve the existing install stderr character tail and ETARGET+ // behavior, including multibyte diagnostics. Only private config capture+ // uses a byte bound; any truncated E404 evidence is rejected.+ const next = previous + chunk.toString();+ outputTruncated ||= next.length > outputLimit;+ return next.slice(-outputLimit);+ };let stopping;const kill = (kind) => {try {@@ -34,15 +161,20 @@ function run(command, args, { env, signal, captureStderr = false }) {};signal.addEventListener("abort", stop, { once: true });child.stderr?.on("data", (chunk) => {- process.stderr.write(chunk);- stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARS);+ if (!quiet) process.stderr.write(chunk);+ stderr = retain(stderr, chunk);+ });+ child.stdout?.on("data", (chunk) => {+ if (!quiet) process.stdout.write(chunk);+ stdout = retain(stdout, chunk);});child.on("error", reject);child.on("close", async (code, exitSignal) => {signal.removeEventListener("abort", stop);await stopping;if (signal.aborted) reject(signal.reason);- else resolve({ code, signal: exitSignal, stderr });+ else resolve({ code, signal: exitSignal,+ stderr: quiet ? stderr.toString("utf8") : stderr, stdout: quiet ? stdout.toString("utf8") : stdout, outputTruncated });});});}@@ -85,7 +217,10 @@ export async function startPublishedCanary({if (result.code === 0) break;const codes = [...result.stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)];const missingVersion = codes.at(-1)?.[1] === "ETARGET";- if (!missingVersion || result.signal || attempt === MAX_INSTALL_ATTEMPTS) {+ const retryable = !result.signal && attempt < MAX_INSTALL_ATTEMPTS+ && (missingVersion || (!result.outputTruncated+ && await canRetryPublishedTarball404(result.stderr, version, { signal: installSignal, prefix, env })));+ if (!retryable) {throw new Error(`Canary npm installation failed (exit ${result.code ?? result.signal}); onboarding was not started`);}await rm(prefix, { recursive: true, force: true });