Skip to content

The open-source app everyone uses to manage agents at work

feat(ci): publish immutable cloud migrator artifacts (#13455)

Devin Foleyda77a0c28c2d
diff --git a/.github/cloud-migrator-deploy/cloudfront-read-statement.json b/.github/cloud-migrator-deploy/cloudfront-read-statement.json
new file mode 100644
index 0000000000..cf36a18d2f
--- /dev/null
+++ b/.github/cloud-migrator-deploy/cloudfront-read-statement.json
@@ -0,0 +1,14 @@
+{
+ "Sid": "AllowCloudFrontReadCloudMigrators",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "cloudfront.amazonaws.com"
+ },
+ "Action": "s3:GetObject",
+ "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/cloud-migrators/v1/*",
+ "Condition": {
+ "StringEquals": {
+ "AWS:SourceArn": "arn:aws:cloudfront::078455283791:distribution/E3GTU28BBO2SFR"
+ }
+ }
+}
diff --git a/.github/cloud-migrator-deploy/trust-policy.json b/.github/cloud-migrator-deploy/trust-policy.json
new file mode 100644
index 0000000000..db8d148aa3
--- /dev/null
+++ b/.github/cloud-migrator-deploy/trust-policy.json
@@ -0,0 +1,18 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Principal": {
+ "Federated": "arn:aws:iam::078455283791:oidc-provider/token.actions.githubusercontent.com"
+ },
+ "Action": "sts:AssumeRoleWithWebIdentity",
+ "Condition": {
+ "StringEquals": {
+ "token.actions.githubusercontent.com:aud": "sts.amazonaws.com",
+ "token.actions.githubusercontent.com:sub": "repo:paperclipai/paperclip:ref:refs/heads/master"
+ }
+ }
+ }
+ ]
+}
diff --git a/.github/cloud-migrator-deploy/upload-policy.json b/.github/cloud-migrator-deploy/upload-policy.json
new file mode 100644
index 0000000000..478ba9d36d
--- /dev/null
+++ b/.github/cloud-migrator-deploy/upload-policy.json
@@ -0,0 +1,25 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "s3:PutObject",
+ "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1/cloud-migrators/v1/*",
+ "Condition": {
+ "StringEquals": {
+ "s3:if-none-match": "*"
+ }
+ }
+ },
+ {
+ "Effect": "Allow",
+ "Action": "s3:ListBucket",
+ "Resource": "arn:aws:s3:::paperclipai-runner-e2e-history-078455283791-us-east-1",
+ "Condition": {
+ "StringLike": {
+ "s3:prefix": "cloud-migrators/v1/*"
+ }
+ }
+ }
+ ]
+}
diff --git a/.github/workflows/cloud-migrator-artifacts.yml b/.github/workflows/cloud-migrator-artifacts.yml
new file mode 100644
index 0000000000..f4da1b25b1
--- /dev/null
+++ b/.github/workflows/cloud-migrator-artifacts.yml
@@ -0,0 +1,90 @@
+name: Cloud migrator artifacts
+run-name: Cloud migrator artifacts ${{ github.sha }}
+
+on:
+ push:
+ branches: [master]
+ workflow_dispatch:
+
+permissions: {}
+concurrency:
+ group: cloud-migrator-artifacts-${{ github.sha }}
+ cancel-in-progress: false
+
+jobs:
+ build:
+ if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ with:
+ version: 9.15.4
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+ - name: Install migrator build dependencies
+ run: pnpm install --ignore-scripts --no-frozen-lockfile --filter @paperclipai/db... --filter @paperclipai/shared...
+ - name: Build exact-source packages and dependency lockfile
+ env:
+ SOURCE_SHA: ${{ github.sha }}
+ run: |
+ node scripts/preview-artifacts.mjs pack . migrator-artifacts "$SOURCE_SHA"
+ node scripts/cloud-migrator-artifacts.mjs build migrator-artifacts "$SOURCE_SHA"
+ node scripts/cloud-migrator-artifacts.mjs verify-install migrator-artifacts "$SOURCE_SHA"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: cloud-migrator-bundle
+ path: |
+ migrator-artifacts/db.tgz
+ migrator-artifacts/shared.tgz
+ migrator-artifacts/package-lock.json
+ migrator-artifacts/manifest.json
+ if-no-files-found: error
+ retention-days: 3
+
+ publish:
+ needs: build
+ if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ contents: read
+ id-token: write
+ attestations: write
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ sparse-checkout: scripts
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: cloud-migrator-bundle
+ path: migrator-artifacts
+ - name: Validate the complete bundle before attestation
+ env:
+ SOURCE_SHA: ${{ github.sha }}
+ run: node scripts/cloud-migrator-artifacts.mjs validate migrator-artifacts "$SOURCE_SHA"
+ - name: Attest the manifest and every content hash it pins
+ uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4
+ with:
+ subject-path: migrator-artifacts/manifest.json
+ - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
+ with:
+ role-to-assume: arn:aws:iam::078455283791:role/paperclip-cloud-migrator-github
+ aws-region: us-east-1
+ role-duration-seconds: 900
+ - name: Publish immutable migrator and verify public downloads
+ env:
+ SOURCE_SHA: ${{ github.sha }}
+ run: node scripts/cloud-migrator-artifacts.mjs publish migrator-artifacts "$SOURCE_SHA"
diff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.md
index 3ffec98887..4157882ef9 100644
--- a/doc/preview-release-artifacts.md
+++ b/doc/preview-release-artifacts.md
@@ -122,3 +122,62 @@ node scripts/preview-artifacts.mjs pack /path/to/source /path/to/output FULL_SHA
This executes source build scripts. Keep output outside the repository and use an
environment without publishing or cloud-admin credentials.
+
+## Direct cloud migrator artifacts
+
+`cloud-migrator-artifacts.yml` builds the DB and shared preview packages on each
+canonical `master` push. A manual run also requires `master` and uses its exact
+commit. This workflow runs on GitHub-hosted runners. It has no PR trigger.
+
+The build resolves a complete npm lockfile from the two local archives. It then
+pins their download URLs to immutable, content-addressed objects. The cloud
+migration runner can use `npm ci` with this lockfile before either new package
+version is available on npm. Existing external dependencies still come from npm
+and carry SHA-512 integrity pins. Package lifecycle scripts remain disabled.
+Before upload, the build job smoke-installs the real archives and their complete
+external and bundled dependency graph with an empty npm cache. It imports both
+installed packages. This check uses local archive URLs because public objects
+do not exist yet; all versions and integrity pins remain unchanged.
+
+Artifacts use the existing runner-history S3 bucket and CloudFront distribution,
+under the separate `cloud-migrators/v1/` prefix. The manifest at
+`https://d1p6rlowie26tp.cloudfront.net/cloud-migrators/v1/<full-sha>/manifest.json`
+records the full source SHA, exact preview version, and the size, URL, and SHA-512
+hash of each archive and the lockfile. Blob URLs include the content hash.
+The publisher validates the complete bundle before any write, writes all blobs
+before the manifest, and verifies downloads through the public endpoint.
+A retry reuses a complete existing manifest after verification. The publisher
+also creates a GitHub/Sigstore build-provenance attestation for the manifest
+before upload. This independently binds all package and lockfile content hashes
+to the canonical workflow, master ref, repository identity, and source commit.
+Cloud must verify this signature and its certificate claims before accepting
+the executable archives; hashes served by the artifact store alone are not
+sufficient provenance.
+
+The build job has no AWS credential. The publish job downloads only the four
+fixed files, validates them, and uploads them without executing their code.
+The dedicated `paperclip-cloud-migrator-github` OIDC role trusts only
+`repo:paperclipai/paperclip:ref:refs/heads/master`. Its policy permits prefix
+listing and conditional `PutObject` calls in this one prefix. It permits no
+object deletion or overwrite. PRs, including allowlisted PRs, cannot assume it.
+
+The deploy policies are checked in under `.github/cloud-migrator-deploy/`:
+
+- `trust-policy.json`: the role trust policy.
+- `upload-policy.json`: the role's inline permission policy.
+- `cloudfront-read-statement.json`: append this statement to the existing bucket
+ policy, preserving its other statements and public-access blocks.
+
+There is no lifecycle expiry on this prefix. Keep referenced artifacts for
+rollback; deleting them can prevent a fresh migrator install for an old release.
+This producer rollout is additive. npm preview publication and the current
+cloud readiness gate remain active until the cloud consumer supports the new
+manifest. A later cutover must preserve source verification, image identity,
+migration compatibility, and the cloud runner's integrity checks.
+
+Local verification:
+
+```sh
+node --test scripts/cloud-migrator-artifacts.test.mjs
+node scripts/cloud-migrator-artifacts.mjs verify <full-sha>
+```
diff --git a/package.json b/package.json
index 9df886d7b7..8d1bdd7a0d 100644
--- a/package.json
+++ b/package.json
@@ -59,7 +59,7 @@
"smoke:posthog-live": "node scripts/smoke/posthog-live.mjs",
"smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh",
"smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",
- "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/select-cloud-cache.test.mjs",
+ "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/select-cloud-cache.test.mjs",
"storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs",
"test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts",
"test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",
diff --git a/scripts/cloud-migrator-artifacts.mjs b/scripts/cloud-migrator-artifacts.mjs
new file mode 100644
index 0000000000..6c097a584d
--- /dev/null
+++ b/scripts/cloud-migrator-artifacts.mjs
@@ -0,0 +1,208 @@
+#!/usr/bin/env node
+// The build job has no publish credential. The publisher only validates and
+// uploads fixed data files; it never installs or executes package code.
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { copyFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { assertMetadata, tarManifest, versionFor } from "./preview-artifacts.mjs";
+
+export const artifactBase = "https://d1p6rlowie26tp.cloudfront.net/cloud-migrators/v1";
+export const artifactBucket = "paperclipai-runner-e2e-history-078455283791-us-east-1";
+const prefix = "cloud-migrators/v1/";
+const names = ["db", "shared"];
+const maximumBytes = 32 * 1024 * 1024;
+const integrityFor = (bytes) => `sha512-${createHash("sha512").update(bytes).digest("base64")}`;
+
+export function descriptor(bytes, extension) {
+ const hash = createHash("sha512").update(bytes).digest("hex");
+ return { url: `${artifactBase}/blobs/${hash}.${extension}`, integrity: integrityFor(bytes), size: bytes.length };
+}
+
+function assertDescriptor(pin, extension) {
+ if (!pin || typeof pin.integrity !== "string" || !/^sha512-[A-Za-z0-9+/]{86}==$/.test(pin.integrity) ||
+ !Number.isSafeInteger(pin.size) || pin.size <= 0 || pin.size > maximumBytes) throw new Error("Invalid artifact integrity or size.");
+ const digest = Buffer.from(pin.integrity.slice(7), "base64");
+ if (digest.toString("base64") !== pin.integrity.slice(7) || pin.url !== `${artifactBase}/blobs/${digest.toString("hex")}.${extension}`) {
+ throw new Error("Artifact URL does not match its content hash and trusted origin.");
+ }
+}
+
+export function assertManifest(manifest, sha) {
+ if (manifest?.version !== 1 || manifest.sourceSha !== sha || manifest.packageVersion !== versionFor(sha)) throw new Error("Artifact source identity mismatch.");
+ for (const name of names) assertDescriptor(manifest.packages?.[name], "tgz");
+ assertDescriptor(manifest.lockfile, "json");
+}
+
+export function assertLockfile(lock, manifest) {
+ const version = manifest.packageVersion;
+ if (lock?.lockfileVersion !== 3 || !lock.packages || Array.isArray(lock.packages) ||
+ JSON.stringify(lock.packages[""]?.dependencies) !== JSON.stringify({ "@paperclipai/db": version })) throw new Error("Invalid migrator lockfile root.");
+ for (const name of names) {
+ const pin = lock.packages[`node_modules/@paperclipai/${name}`];
+ const expected = manifest.packages[name];
+ if (pin?.version !== version || pin.integrity !== expected.integrity || pin.resolved !== expected.url || pin.link || pin.inBundle) throw new Error("Migrator lockfile package pin mismatch.");
+ }
+ if (lock.packages["node_modules/@paperclipai/db"].dependencies?.["@paperclipai/shared"] !== version) throw new Error("Migrator shared dependency mismatch.");
+ for (const [key, entry] of Object.entries(lock.packages)) {
+ if (key === "") continue;
+ if (!entry || typeof entry !== "object" || entry.link) throw new Error("Invalid migrator lockfile entry.");
+ if (/(?:^|\/)node_modules\/@paperclipai\/[^/]+$/.test(key) && !names.some((name) => key === `node_modules/@paperclipai/${name}`)) throw new Error("Unexpected internal migrator dependency.");
+ if (entry.inBundle === true) {
+ if (!key.startsWith("node_modules/@paperclipai/db/node_modules/")) throw new Error("Unexpected bundled dependency.");
+ continue;
+ }
+ if (!/^sha512-[A-Za-z0-9+/]{86}==$/.test(entry.integrity ?? "")) throw new Error("Migrator dependency has no strong integrity pin.");
+ if (names.some((name) => key === `node_modules/@paperclipai/${name}`)) continue;
+ const url = new URL(entry.resolved);
+ if (url.origin !== "https://registry.npmjs.org" || url.username || url.password || url.search || url.hash) throw new Error("Migrator dependency must resolve to npm.");
+ }
+}
+
+export function buildBundle(directory, sha, { exec = execFileSync } = {}) {
+ versionFor(sha);
+ directory = path.resolve(directory);
+ const packages = {};
+ for (const name of names) {
+ const bytes = readFileSync(path.join(directory, `${name}.tgz`));
+ assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha);
+ packages[name] = descriptor(bytes, "tgz");
+ }
+ const scratch = mkdtempSync(path.join(os.tmpdir(), "cloud-migrator-lock-"));
+ try {
+ for (const name of names) copyFileSync(path.join(directory, `${name}.tgz`), path.join(scratch, `${name}.tgz`));
+ const root = { name: "paperclip-migrator-install-root", version: "0.0.0", private: true,
+ dependencies: { "@paperclipai/db": "file:db.tgz", "@paperclipai/shared": "file:shared.tgz" } };
+ writeFileSync(path.join(scratch, "package.json"), JSON.stringify(root));
+ exec("npm", ["install", "--package-lock-only", "--ignore-scripts", "--no-audit", "--no-fund", "--registry=https://registry.npmjs.org"], { cwd: scratch, stdio: "inherit", timeout: 180_000 });
+ const lock = JSON.parse(readFileSync(path.join(scratch, "package-lock.json"), "utf8"));
+ // Both new packages are local during resolution. npm ci subsequently uses
+ // these immutable URLs, without looking up the new npm versions.
+ lock.packages[""].dependencies = { "@paperclipai/db": versionFor(sha) };
+ for (const name of names) lock.packages[`node_modules/@paperclipai/${name}`].resolved = packages[name].url;
+ const lockBytes = Buffer.from(JSON.stringify(lock) + "\n");
+ const manifest = { version: 1, sourceSha: sha, packageVersion: versionFor(sha), packages, lockfile: descriptor(lockBytes, "json") };
+ assertManifest(manifest, sha);
+ assertLockfile(lock, manifest);
+ writeFileSync(path.join(directory, "package-lock.json"), lockBytes);
+ writeFileSync(path.join(directory, "manifest.json"), JSON.stringify(manifest) + "\n");
+ return manifest;
+ } finally { rmSync(scratch, { recursive: true, force: true }); }
+}
+
+function verifyBytes(bytes, pin) {
+ if (bytes.length !== pin.size || integrityFor(bytes) !== pin.integrity) throw new Error("Artifact bytes do not match their immutable pin.");
+}
+
+export function validateBundle(directory, sha) {
+ const manifest = JSON.parse(readFileSync(path.join(directory, "manifest.json"), "utf8"));
+ assertManifest(manifest, sha);
+ for (const name of names) {
+ const bytes = readFileSync(path.join(directory, `${name}.tgz`));
+ verifyBytes(bytes, manifest.packages[name]);
+ assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha);
+ }
+ const bytes = readFileSync(path.join(directory, "package-lock.json"));
+ verifyBytes(bytes, manifest.lockfile);
+ assertLockfile(JSON.parse(bytes), manifest);
+ return manifest;
+}
+
+/** Exercise the real dependency graph before publishing, with no new npm versions. */
+export function verifyInstall(directory, sha, { exec = execFileSync } = {}) {
+ const manifest = validateBundle(directory, sha);
+ const scratch = mkdtempSync(path.join(os.tmpdir(), "cloud-migrator-install-"));
+ try {
+ const lock = JSON.parse(readFileSync(path.join(directory, "package-lock.json"), "utf8"));
+ for (const name of names) {
+ copyFileSync(path.join(directory, `${name}.tgz`), path.join(scratch, `${name}.tgz`));
+ // The public objects do not exist yet. Only transport changes for this
+ // smoke install; exact versions, integrity, root and transitive pins stay.
+ lock.packages[`node_modules/@paperclipai/${name}`].resolved = `file:${name}.tgz`;
+ }
+ writeFileSync(path.join(scratch, "package.json"), JSON.stringify({ name: "paperclip-migrator-install-root", version: "0.0.0", private: true,
+ dependencies: { "@paperclipai/db": manifest.packageVersion } }));
+ writeFileSync(path.join(scratch, "package-lock.json"), JSON.stringify(lock));
+ exec("npm", ["ci", "--ignore-scripts", "--no-audit", "--no-fund", "--update-notifier=false", "--cache", path.join(scratch, "empty-cache"),
+ "--registry=https://registry.npmjs.org"], { cwd: scratch, stdio: "inherit", timeout: 180_000 });
+ for (const name of names) assertMetadata(JSON.parse(readFileSync(path.join(scratch, "node_modules", "@paperclipai", name, "package.json"), "utf8")), `@paperclipai/${name}`, sha);
+ exec(process.execPath, ["--input-type=module", "--eval", "await import('@paperclipai/db'); await import('@paperclipai/shared');"], { cwd: scratch, stdio: "inherit", timeout: 30_000 });
+ } finally { rmSync(scratch, { recursive: true, force: true }); }
+}
+
+async function download(url, fetchImpl) {
+ const response = await fetchImpl(url, { redirect: "error", signal: AbortSignal.timeout(60_000) });
+ if (!response.ok) throw new Error(`Artifact download failed: HTTP ${response.status}`, { cause: { status: response.status } });
+ const reader = response.body.getReader();
+ const chunks = [];
+ let size = 0;
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ size += value.length;
+ if (size > maximumBytes) throw new Error("Artifact exceeds size limit.");
+ chunks.push(value);
+ }
+ } finally { await reader.cancel(); }
+ return Buffer.concat(chunks);
+}
+
+export async function verifyPublished(sha, fetchImpl = fetch) {
+ versionFor(sha);
+ const manifest = JSON.parse(await download(`${artifactBase}/${sha}/manifest.json`, fetchImpl));
+ assertManifest(manifest, sha);
+ await Promise.all(names.map(async (name) => {
+ const bytes = await download(manifest.packages[name].url, fetchImpl);
+ verifyBytes(bytes, manifest.packages[name]);
+ assertMetadata(tarManifest(bytes), `@paperclipai/${name}`, sha);
+ }));
+ const lock = await download(manifest.lockfile.url, fetchImpl);
+ verifyBytes(lock, manifest.lockfile);
+ assertLockfile(JSON.parse(lock), manifest);
+ return manifest;
+}
+
+export async function publishBundle(directory, sha, { exec = execFileSync, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) {
+ const manifest = validateBundle(directory, sha);
+ const key = `${prefix}${sha}/manifest.json`;
+ const verifyVisible = async () => {
+ for (let attempt = 0; ; attempt++) {
+ try { return await verifyPublished(sha, fetchImpl); }
+ catch (error) {
+ // A consumer may have cached a missing-object response just before
+ // publication. Wait through the CDN error TTL, never through bad bytes.
+ if (attempt >= 6 || ![403, 404].includes(error.cause?.status)) throw error;
+ await sleep(2_000);
+ }
+ }
+ };
+ const aws = (args) => exec("aws", ["s3api", ...args, "--bucket", artifactBucket, "--region", "us-east-1"], { encoding: "utf8", maxBuffer: 1024 * 1024 });
+ // Exact prefix listing distinguishes missing objects from permission errors.
+ const exists = (objectKey) => JSON.parse(aws(["list-objects-v2", "--prefix", objectKey, "--max-keys", "1"])).Contents?.some((object) => object.Key === objectKey);
+ if (exists(key)) return verifyVisible();
+ const upload = (file, objectKey, contentType) => {
+ if (exists(objectKey)) return;
+ aws(["put-object", "--key", objectKey, "--body", path.resolve(directory, file), "--content-type", contentType,
+ "--cache-control", "public,max-age=31536000,immutable", "--if-none-match", "*"]);
+ };
+ for (const name of names) upload(`${name}.tgz`, prefix + manifest.packages[name].url.slice(`${artifactBase}/`.length), "application/gzip");
+ upload("package-lock.json", prefix + manifest.lockfile.url.slice(`${artifactBase}/`.length), "application/json");
+ // Publish the commit marker last; readers can never observe a partial bundle.
+ upload("manifest.json", key, "application/json");
+ return verifyVisible();
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [command, directory, sha] = process.argv.slice(2);
+ try {
+ if (command === "build") buildBundle(directory, sha);
+ else if (command === "validate") validateBundle(directory, sha);
+ else if (command === "verify-install") verifyInstall(directory, sha);
+ else if (command === "publish") await publishBundle(directory, sha);
+ else if (command === "verify") await verifyPublished(directory);
+ else throw new Error("Expected build, validate, verify-install, publish, or verify.");
+ } catch (error) { console.error(error.message); process.exitCode = 1; }
+}
diff --git a/scripts/cloud-migrator-artifacts.test.mjs b/scripts/cloud-migrator-artifacts.test.mjs
new file mode 100644
index 0000000000..61374f3206
--- /dev/null
+++ b/scripts/cloud-migrator-artifacts.test.mjs
@@ -0,0 +1,144 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { createServer } from "node:http";
+import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { gzipSync } from "node:zlib";
+import { artifactBase, assertManifest, assertLockfile, buildBundle, descriptor, validateBundle, verifyPublished, publishBundle } from "./cloud-migrator-artifacts.mjs";
+import { previewManifest, versionFor } from "./preview-artifacts.mjs";
+
+const sha = "a".repeat(40);
+function fixture(t) {
+ const dir = mkdtempSync(path.join(os.tmpdir(), "migrator-artifact-test-"));
+ t.after(() => rmSync(dir, { recursive: true, force: true }));
+ for (const name of ["db", "shared"]) {
+ const bytes = Buffer.from(JSON.stringify(previewManifest({ name: `@paperclipai/${name}`, dependencies: {} }, sha)));
+ const header = Buffer.alloc(512);
+ header.write("package/package.json"); header.write(bytes.length.toString(8).padStart(11, "0"), 124, 11); header[156] = 48;
+ // A real tar header, so npm can install this fixture as well as inspect it.
+ header.fill(32, 148, 156);
+ const sum = header.reduce((a, b) => a + b, 0);
+ header.write(sum.toString(8).padStart(6, "0") + "\0 ", 148, 8);
+ const padded = Buffer.alloc(Math.ceil(bytes.length / 512) * 512); bytes.copy(padded);
+ writeFileSync(path.join(dir, `${name}.tgz`), gzipSync(Buffer.concat([header, padded, Buffer.alloc(1024)])));
+ }
+ const manifest = buildBundle(dir, sha, { exec: (cmd, args, options) => {
+ assert.equal(cmd, "npm"); assert.ok(args.includes("--ignore-scripts"));
+ const localRoot = JSON.parse(readFileSync(path.join(options.cwd, "package.json")));
+ assert.deepEqual(localRoot.dependencies, { "@paperclipai/db": "file:db.tgz", "@paperclipai/shared": "file:shared.tgz" });
+ const packages = { "": localRoot };
+ for (const name of ["db", "shared"]) packages[`node_modules/@paperclipai/${name}`] = {
+ version: versionFor(sha), integrity: descriptor(readFileSync(path.join(dir, `${name}.tgz`)), "tgz").integrity,
+ resolved: `file:${name}.tgz`, ...(name === "db" ? { dependencies: { "@paperclipai/shared": versionFor(sha) } } : {}),
+ };
+ writeFileSync(path.join(options.cwd, "package-lock.json"), JSON.stringify({ lockfileVersion: 3, packages }));
+ } });
+ const files = new Map([[`${artifactBase}/${sha}/manifest.json`, readFileSync(path.join(dir, "manifest.json"))]]);
+ for (const name of ["db", "shared"]) files.set(manifest.packages[name].url, readFileSync(path.join(dir, `${name}.tgz`)));
+ files.set(manifest.lockfile.url, readFileSync(path.join(dir, "package-lock.json")));
+ const fetchImpl = async (url, options) => {
+ assert.equal(options.redirect, "error"); assert.ok(options.signal);
+ assert.ok(files.has(url), `unexpected download: ${url}`);
+ return new Response(files.get(url));
+ };
+ return { dir, manifest, files, fetchImpl };
+}
+
+test("bundle pins the exact source pair and complete lockfile without new npm lookups", async (t) => {
+ const { dir, manifest, fetchImpl } = fixture(t);
+ assert.deepEqual(validateBundle(dir, sha), manifest);
+ assert.deepEqual(await verifyPublished(sha, fetchImpl), manifest);
+});
+
+test("source identity, content hashes, size, and origin fail closed", async (t) => {
+ const { dir, manifest, files, fetchImpl } = fixture(t);
+ for (const mutate of [
+ (m) => { m.sourceSha = "b".repeat(40); },
+ (m) => { m.packages.db.url = "https://evil.invalid/db.tgz"; },
+ (m) => { m.packages.shared.size = 0; },
+ (m) => { m.lockfile.integrity = "sha1-weak"; },
+ ]) {
+ const bad = structuredClone(manifest); mutate(bad); assert.throws(() => assertManifest(bad, sha));
+ }
+ files.set(manifest.packages.db.url, Buffer.from("corrupt"));
+ await assert.rejects(verifyPublished(sha, fetchImpl), /immutable pin/);
+ writeFileSync(path.join(dir, "db.tgz"), "corrupt");
+ await assert.rejects(publishBundle(dir, sha, { exec: () => assert.fail("no upload before pair validation") }));
+});
+
+test("lockfile rejects mutable, foreign, linked, and mismatched dependencies", (t) => {
+ const { dir, manifest } = fixture(t);
+ const lock = JSON.parse(readFileSync(path.join(dir, "package-lock.json")));
+ for (const mutate of [
+ (l) => { l.packages[""].dependencies["@paperclipai/db"] = "latest"; },
+ (l) => { l.packages["node_modules/@paperclipai/shared"].version = "0.0.0"; },
+ (l) => { l.packages["node_modules/@paperclipai/db"].link = true; },
+ (l) => { l.packages["node_modules/evil"] = { inBundle: true }; },
+ (l) => { l.packages["node_modules/evil"] = { integrity: manifest.packages.db.integrity, resolved: "https://evil.invalid/pkg.tgz" }; },
+ (l) => { l.packages["node_modules/evil"] = { integrity: "sha1-weak", resolved: "https://registry.npmjs.org/pkg.tgz" }; },
+ (l) => { l.packages["node_modules/a/node_modules/@paperclipai/shared"] = l.packages["node_modules/@paperclipai/shared"]; },
+ ]) {
+ const bad = structuredClone(lock); mutate(bad); assert.throws(() => assertLockfile(bad, manifest));
+ }
+});
+
+test("publisher writes blobs first, marker last, and never overwrites existing objects", async (t) => {
+ const { dir, fetchImpl } = fixture(t);
+ const objects = new Set(); const uploads = [];
+ const exec = (cmd, args) => {
+ assert.equal(cmd, "aws");
+ const arg = (key) => args[args.indexOf(key) + 1];
+ if (args[1] === "list-objects-v2") return JSON.stringify({ Contents: objects.has(arg("--prefix")) ? [{ Key: arg("--prefix") }] : [] });
+ assert.equal(args[1], "put-object"); assert.equal(arg("--if-none-match"), "*");
+ objects.add(arg("--key")); uploads.push(arg("--key")); return "{}";
+ };
+ await publishBundle(dir, sha, { exec, fetchImpl });
+ assert.equal(uploads.length, 4); assert.equal(uploads.at(-1), `cloud-migrators/v1/${sha}/manifest.json`);
+ await publishBundle(dir, sha, { exec, fetchImpl }); assert.equal(uploads.length, 4);
+});
+
+test("download failures and oversized objects never count as available", async (t) => {
+ fixture(t);
+ for (const status of [403, 404, 500]) await assert.rejects(verifyPublished(sha, async () => new Response(null, { status })), /download failed/);
+ await assert.rejects(verifyPublished(sha, async () => new Response(Buffer.alloc(32 * 1024 * 1024 + 1))), /size limit/);
+});
+
+test("real npm ci installs the new pair from pinned archives with an empty cache", async (t) => {
+ const { dir } = fixture(t);
+ // Real npm resolution uses local archives; neither package version exists on npm.
+ const manifest = buildBundle(dir, sha);
+ const lock = JSON.parse(readFileSync(path.join(dir, "package-lock.json")));
+ const requests = [];
+ const server = createServer((req, res) => {
+ requests.push(req.url);
+ if (!["/db.tgz", "/shared.tgz"].includes(req.url)) { res.writeHead(500); res.end(); return; }
+ res.end(readFileSync(path.join(dir, req.url.slice(1))));
+ });
+ await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+ t.after(() => new Promise((resolve) => server.close(resolve)));
+ const base = `http://127.0.0.1:${server.address().port}`;
+ for (const name of ["db", "shared"]) lock.packages[`node_modules/@paperclipai/${name}`].resolved = `${base}/${name}.tgz`;
+ writeFileSync(path.join(dir, "package.json"), JSON.stringify({ name: "paperclip-migrator-install-root", version: "0.0.0", private: true, dependencies: { "@paperclipai/db": versionFor(sha) } }));
+ writeFileSync(path.join(dir, "package-lock.json"), JSON.stringify(lock));
+ await promisify(execFile)("npm", ["ci", "--update-notifier=false", "--ignore-scripts", "--no-audit", "--no-fund", "--registry", base, "--cache", path.join(dir, "empty-cache")], { cwd: dir, timeout: 60_000 });
+ assert.deepEqual(requests.sort(), ["/db.tgz", "/shared.tgz"]);
+ for (const name of ["db", "shared"]) assert.equal(JSON.parse(readFileSync(path.join(dir, `node_modules/@paperclipai/${name}/package.json`))).version, manifest.packageVersion);
+});
+
+test("AWS trust is master-only and publication policy cannot overwrite objects", () => {
+ const read = (name) => JSON.parse(readFileSync(new URL(`../.github/cloud-migrator-deploy/${name}.json`, import.meta.url)));
+ assert.equal(read("trust-policy").Statement[0].Condition.StringEquals["token.actions.githubusercontent.com:sub"], "repo:paperclipai/paperclip:ref:refs/heads/master");
+ const policy = read("upload-policy").Statement;
+ assert.deepEqual(policy.map((s) => s.Action), ["s3:PutObject", "s3:ListBucket"]);
+ assert.equal(policy[0].Condition.StringEquals["s3:if-none-match"], "*");
+ const workflow = readFileSync(new URL("../.github/workflows/cloud-migrator-artifacts.yml", import.meta.url), "utf8");
+ assert.ok(!workflow.includes("pull_request") && !workflow.includes("self-hosted") && !workflow.includes("runs-on/fleet="));
+ assert.equal((workflow.match(/id-token: write/g) ?? []).length, 1);
+ assert.equal((workflow.match(/attestations: write/g) ?? []).length, 1);
+ assert.ok(workflow.indexOf(" validate migrator-artifacts") < workflow.indexOf("uses: actions/attest@"));
+ assert.ok(workflow.indexOf("uses: actions/attest@") < workflow.indexOf(" publish migrator-artifacts"));
+ assert.ok(workflow.indexOf(" verify-install migrator-artifacts") < workflow.indexOf("actions/upload-artifact@"), "the real dependency smoke must pass before artifact upload");
+});