The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
Build isolated preview artifacts for exact-source deployments (#13041)
Dotta0cc796b7bdbd
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 3621bc98c9..49c00e59fd 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -1,4 +1,5 @@name: Release+run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || 'Release' }}on:push:@@ -17,12 +18,21 @@ on:- stable- beta- nightly+ - previewdefault: stablesource_ref:- description: (stable) Commit SHA, branch, or tag to publish as stable+ description: Stable source ref, or full immutable SHA for a preview buildrequired: truetype: stringdefault: master+ request_id:+ description: (preview) CLI correlation UUID+ type: string+ default: ""+ preview_migrator:+ description: (preview) Publish isolated shared and database packages if missing+ type: boolean+ default: falsestable_date:description: Enter a UTC date in YYYY-MM-DD format, for example 2026-03-18. Do not enter a version string. The workflow will resolve that date to a stable version such as 2026.318.0, then 2026.318.1 for the next same-day stable.required: false@@ -46,7 +56,7 @@ on:default: falseconcurrency:- group: release-${{ github.event_name }}-${{ github.ref }}+ group: ${{ inputs.channel == 'preview' && format('preview-{0}', inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }}cancel-in-progress: falseenv:@@ -64,6 +74,219 @@ env:NPM_PUBLISH_VERIFY_DELAY_SECONDS: "10"jobs:+ plan_preview:+ name: Check preview artifacts+ if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && inputs.channel == 'preview' && !inputs.dry_run+ runs-on: ubuntu-latest+ permissions:+ contents: read+ outputs:+ image: ${{ steps.plan.outputs.image }}+ packages: ${{ steps.plan.outputs.packages }}+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ persist-credentials: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - name: Validate immutable source and inspect existing artifacts+ id: plan+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ REQUEST_ID: ${{ inputs.request_id }}+ PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}+ run: node scripts/preview-artifacts.mjs plan "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR"++ package_preview:+ name: Build preview migrator+ needs: plan_preview+ if: needs.plan_preview.outputs.packages == 'true'+ runs-on: ubuntu-latest+ timeout-minutes: 30+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ path: trusted+ persist-credentials: false+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ inputs.source_ref }}+ path: source+ persist-credentials: false+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6+ with:+ version: 9.15.4+ run_install: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - name: Install build dependencies without lifecycle scripts+ working-directory: source+ run: pnpm install --ignore-scripts --no-frozen-lockfile+ - name: Build and pack exact-source preview packages+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ run: node trusted/scripts/preview-artifacts.mjs pack source packages "$SOURCE_SHA"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: preview-packages+ overwrite: true+ path: packages/*.tgz+ if-no-files-found: error+ retention-days: 7++ publish_preview:+ name: Publish preview migrator+ needs: [plan_preview, package_preview]+ if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success'+ runs-on: ubuntu-latest+ timeout-minutes: 30+ # Reuse release.yml's established npm trusted-publisher identity. This job+ # publishes only isolated preview versions; it cannot advance lane tags.+ environment: npm-canary+ permissions:+ contents: read+ id-token: write+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ persist-credentials: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - name: Install npm with trusted publishing support+ run: npm install --global [email protected] --ignore-scripts+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: preview-packages+ path: preview-packages+ - name: Publish immutable preview packages without running package code+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ run: node scripts/preview-artifacts.mjs publish preview-packages "$SOURCE_SHA"++ image_preview:+ name: Build preview cloud image+ needs: plan_preview+ if: needs.plan_preview.outputs.image == 'true'+ runs-on: ubuntu-latest+ timeout-minutes: 60+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ ref: ${{ inputs.source_ref }}+ persist-credentials: false+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6+ with:+ version: 9.15.4+ run_install: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - name: Prepare locked image context+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ run: |+ set -euo pipefail+ test "$(git rev-parse HEAD)" = "$SOURCE_SHA"+ pnpm install --resolution-only --ignore-scripts --ignore-pnpmfile --no-frozen-lockfile+ echo "TOOLS_EPOCH=$(date -u +%G-W%V)" >> "$GITHUB_ENV"+ - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4+ - name: Build the immutable cloud image without registry credentials+ uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7+ with:+ context: .+ target: cloud+ platforms: linux/amd64+ push: false+ provenance: false # Docker archives cannot carry registry attestations.+ outputs: type=docker,dest=${{ runner.temp }}/preview-image.tar+ tags: ghcr.io/paperclipai/paperclip:sha-${{ inputs.source_ref }}-cloud+ build-args: |+ CLOUD_BUNDLED_PLUGINS=daytona+ CLOUD_BUNDLED_SERVER_DEPS=@sentry/node+ PAPERCLIP_BUILD_COMMIT=${{ inputs.source_ref }}+ PAPERCLIP_BUILD_VERSION=0.0.0-preview.g${{ inputs.source_ref }}+ CLI_TOOLS_CACHE_EPOCH=${{ env.TOOLS_EPOCH }}+ labels: |+ org.opencontainers.image.revision=${{ inputs.source_ref }}+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: preview-image+ overwrite: true+ path: ${{ runner.temp }}/preview-image.tar+ compression-level: 0+ if-no-files-found: error+ retention-days: 1++ publish_image_preview:+ name: Publish preview cloud image+ needs: [plan_preview, image_preview]+ if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.image == 'true' && needs.image_preview.result == 'success'+ runs-on: ubuntu-latest+ timeout-minutes: 30+ # This existing environment has an external master-only branch policy.+ environment: npm-canary+ permissions:+ contents: read+ packages: write+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ persist-credentials: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: preview-image+ path: preview-image+ - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4+ with:+ registry: ghcr.io+ username: ${{ github.repository_owner }}+ password: ${{ secrets.GITHUB_TOKEN }}+ - name: Verify image identity and publish without executing image code+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ run: node scripts/preview-artifacts.mjs publish-image preview-image/preview-image.tar "$SOURCE_SHA"++ result_preview:+ name: Verify preview artifacts+ needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview]+ if: >-+ always() && needs.plan_preview.result == 'success' &&+ (needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') &&+ (needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false')+ runs-on: ubuntu-latest+ permissions:+ contents: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ persist-credentials: false+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7+ with:+ node-version: 24+ - name: Confirm exact artifacts are visible+ env:+ SOURCE_SHA: ${{ inputs.source_ref }}+ REQUEST_ID: ${{ inputs.request_id }}+ PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}+ run: node scripts/preview-artifacts.mjs result "$SOURCE_SHA" "$REQUEST_ID"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: stack-deploy-result+ overwrite: true+ path: stack-deploy-result/result.json+ if-no-files-found: error+ retention-days: 30+verify_canary:if: github.event_name == 'push'uses: ./.github/workflows/release-verify.ymldiff --git a/doc/preview-release-artifacts.md b/doc/preview-release-artifacts.mdnew file mode 100644index 0000000000..11f0c2808d--- /dev/null+++ b/doc/preview-release-artifacts.md@@ -0,0 +1,93 @@+# Preview deployment artifacts++The `preview` channel in `.github/workflows/release.yml` builds deployment+artifacts for one immutable source commit. It does not create a GitHub release,+move a source branch, or advance any stable, beta, nightly, or canary alias.++Dispatch `release.yml` on `master` with these inputs:++| Input | Value |+| --- | --- |+| `channel` | `preview` |+| `source_ref` | Full lowercase 40-character commit SHA in this repository |+| `request_id` | UUID v4 identifying the operator's deployment request |+| `preview_migrator` | `true` when exact-source DB/shared packages are needed |+| `dry_run` | `false` |++The workflow title is `Stack deploy <request_id> build`. Consumers must find a+run by this identity, not by the latest run. Preview builds reject workflow+definitions that do not run from `master`.++## Outputs and reuse++The image uses `ghcr.io/paperclipai/paperclip:sha-<FULL_SHA>-cloud`.+Full-SHA tags keep separate commits with the same short prefix isolated. Normal+release images retain their existing short-tag convention. Build arguments carry the full commit SHA.+Preview builds do not import or overwrite the shared release cache or release+aliases. Missing images are built for Linux amd64, matching managed deployments.++When requested, both `@paperclipai/shared` and `@paperclipai/db` use+`0.0.0-preview.g<FULL_SHA>`. Workspace dependencies are pinned to exact versions.+Packages carry `gitHead` and `paperclipPreviewCommit` source identity. npm publishes+them under the `preview` dist-tag only. Normal consumers of `latest` or `canary`+continue to select normal releases.++Registry 404 responses mean missing artifacts. Authentication errors, outages,+or existing package identity mismatches fail the workflow. Retries reuse matching+published artifacts, including a shared package published before a DB publish+failure. Allow npm's visibility polling to finish before retrying.++The final `stack-deploy-result` artifact contains `result.json` with contract+version 1, request ID, SHA, stage `build`, and status `ready`. It expires after+30 days. This confirms artifact availability; it does not certify a tenant deploy.++## Publishing configuration and isolation++Configure npm trusted publishing for **both packages** with repository+`paperclipai/paperclip`, workflow `release.yml`, and environment `npm-canary`.+The image publisher uses the same environment, whose deployment branch policy+permits only master. Both publishers also check the workflow ref before running.+This uses the existing publisher identity rather than requiring another workflow+registration. The job uses npm with OIDC trusted publishing support and provenance.+The environment's existing protections still apply.++Package compilation runs in a separate job with read-only repository access and+no npm, cloud-admin, or provider credentials. Trusted tooling from `master` packs+the requested source checkout. The existing bundled-package helper takes patch+configuration from that source checkout. Build artifacts contain only the two+fixed package tarballs.++Publishing runs on a fresh runner with trusted tooling, without a checkout of+the requested branch. It checks package identity and exact dependencies, rejects+archive path aliases, and publishes with lifecycle scripts disabled and an explicit+registry and dist-tag. Image builds also run without registry write access and export a Docker archive.+A separate trusted publisher loads that archive as data, verifies its full revision,+platform, and image ID, then pushes only the SHA tag. It never runs the image.+Dependency resolution disables scripts and pnpmfile hooks. Preview actions are+pinned to full commit SHAs.++The deploying control plane must independently verify package integrity, source+identity, SQL and migration journal contents, and schema compatibility. Publish+this workflow support before enabling an operator CLI that depends on it. Test+normal release selection and a new migration-bearing branch in staging before+allowing production use.++## Local checks++```sh+node --test scripts/preview-artifacts.test.mjs+pnpm test:release-registry+pnpm -r typecheck+pnpm test:run+pnpm build+```++For a package build without publishing, install dependencies in a disposable+checkout at an exact commit, then use the trusted helper:++```sh+node scripts/preview-artifacts.mjs pack /path/to/source /path/to/output FULL_SHA+```++This executes source build scripts. Keep output outside the repository and use an+environment without publishing or cloud-admin credentials.diff --git a/package.json b/package.jsonindex 309517bd71..9d7f21654b 100644--- a/package.json+++ b/package.json@@ -59,7 +59,7 @@"smoke:posthog-live": "node scripts/smoke/posthog-live.mjs","smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh","smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",- "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs",+ "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs","storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs","test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts","test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",diff --git a/scripts/prepare-bundled-package.mjs b/scripts/prepare-bundled-package.mjsindex 891da00b71..aeb2bef0c5 100644--- a/scripts/prepare-bundled-package.mjs+++ b/scripts/prepare-bundled-package.mjs@@ -116,8 +116,8 @@ export function selectBundledDependencyPatches(return selectedPatches;}-export function applyBundledDependencyPatches(destinationDir, bundledDependencies) {- const rootPackage = JSON.parse(readFileSync(resolve(repoRoot, "package.json"), "utf8"));+export function applyBundledDependencyPatches(destinationDir, bundledDependencies, sourceRoot = repoRoot) {+ const rootPackage = JSON.parse(readFileSync(resolve(sourceRoot, "package.json"), "utf8"));const patchedDependencies = rootPackage.pnpm?.patchedDependencies ?? {};for (const { packageName, patchPath } of selectBundledDependencyPatches(@@ -129,14 +129,14 @@ export function applyBundledDependencyPatches(destinationDir, bundledDependencie"patch",["-p1", "--forward", "-d", resolve(destinationDir, "node_modules", packageName)],{- input: readFileSync(resolve(repoRoot, patchPath)),+ input: readFileSync(resolve(sourceRoot, patchPath)),stdio: ["pipe", "inherit", "inherit"],},);}}-export function prepareBundledPackage(sourceDir, destinationDir) {+export function prepareBundledPackage(sourceDir, destinationDir, { sourceRoot = repoRoot } = {}) {const sourcePackagePath = resolve(sourceDir, "package.json");const sourcePackage = JSON.parse(readFileSync(sourcePackagePath, "utf8"));const bundledDependencies = sourcePackage.bundleDependencies ?? sourcePackage.bundledDependencies ?? [];@@ -166,7 +166,7 @@ export function prepareBundledPackage(sourceDir, destinationDir) {{ cwd: destinationDir, stdio: "inherit" },);writeFileSync(deployedPackagePath, `${JSON.stringify(publishManifest, null, 2)}\n`);- applyBundledDependencyPatches(destinationDir, bundledDependencies);+ applyBundledDependencyPatches(destinationDir, bundledDependencies, sourceRoot);if (bundledDependencies.includes("acpx")) {const acpxPackage = JSON.parse(diff --git a/scripts/preview-artifacts.mjs b/scripts/preview-artifacts.mjsnew file mode 100644index 0000000000..6a68f83492--- /dev/null+++ b/scripts/preview-artifacts.mjs@@ -0,0 +1,200 @@+#!/usr/bin/env node+// Trusted release tooling. Packaging runs without publish credentials; publishing+// accepts only the two fixed package artifacts and never executes their scripts.+import { execFileSync } from "node:child_process";+import { readFileSync, writeFileSync, mkdirSync, cpSync, renameSync, appendFileSync } from "node:fs";+import path from "node:path";+import { pathToFileURL } from "node:url";+import { gunzipSync } from "node:zlib";+import { createHash } from "node:crypto";+import { materializePublishManifest, prepareBundledPackage } from "./prepare-bundled-package.mjs";++export const versionFor = (sha) => {+ if (!/^[0-9a-f]{40}$/.test(sha ?? "")) throw new Error("Preview builds require a full immutable commit SHA.");+ return `0.0.0-preview.g${sha}`;+};+export function validateRequest(sha, requestId) {+ versionFor(sha);+ if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(requestId ?? "")) throw new Error("A correlation UUID is required.");+}++export function previewManifest(pkg, sha) {+ if (!["@paperclipai/shared", "@paperclipai/db"].includes(pkg.name)) throw new Error("Unexpected preview package.");+ const version = versionFor(sha);+ const exact = structuredClone(pkg);+ for (const section of ["dependencies", "optionalDependencies", "peerDependencies"]) {+ for (const [name, specifier] of Object.entries(exact[section] ?? {})) {+ if (typeof specifier === "string" && specifier.startsWith("workspace:")) exact[section][name] = version;+ }+ }+ const result = materializePublishManifest({ ...exact, version });+ result.gitHead = sha;+ result.paperclipPreviewCommit = sha;+ if (pkg.name === "@paperclipai/db") result.dependencies = { ...result.dependencies, "@paperclipai/shared": version };+ return result;+}++export function assertMetadata(pkg, name, sha) {+ if (pkg?.publishConfig !== undefined || pkg?.name !== name || pkg.version !== versionFor(sha) || pkg.gitHead !== sha || pkg.paperclipPreviewCommit !== sha ||+ (name === "@paperclipai/db" && pkg.dependencies?.["@paperclipai/shared"] !== versionFor(sha))) {+ throw new Error("Preview package identity or dependency pin mismatch.");+ }+}++export function tarManifest(bytes) {+ const tar = gunzipSync(bytes, { maxOutputLength: 128 * 1024 * 1024 });+ let manifest;+ for (let offset = 0; offset + 512 <= tar.length;) {+ const h = tar.subarray(offset, offset + 512);+ if (h.every((v) => v === 0)) break;+ const field = (start, size) => h.subarray(start, start + size).toString("utf8").split("\0")[0].trim();+ const sizeText = field(124, 12);+ if (!/^[0-7]+$/.test(sizeText)) throw new Error("Invalid package archive.");+ const size = Number.parseInt(sizeText, 8);+ if (offset + 512 + size > tar.length) throw new Error("Truncated package archive.");+ const name = `${field(345, 155) ? field(345, 155) + "/" : ""}${field(0, 100)}`;+ if (!name.startsWith("package/") || name.split("/").some((part) => part === "." || part === "..") || ![0, 48, 53].includes(h[156])) throw new Error("Unsupported package archive entry.");+ if (name === "package/package.json") {+ if (manifest || ![0, 48].includes(h[156])) throw new Error("Invalid package manifest entry.");+ manifest = JSON.parse(tar.subarray(offset + 512, offset + 512 + size).toString("utf8"));+ }+ offset += 512 + Math.ceil(size / 512) * 512;+ }+ if (!manifest) throw new Error("Missing package manifest.");+ return manifest;+}++export async function packageExists(name, sha, fetchImpl = fetch) {+ const response = await fetchImpl(`https://registry.npmjs.org/${encodeURIComponent(name)}/${versionFor(sha)}`, { signal: AbortSignal.timeout(30_000) });+ if (response.status === 404) return false;+ if (!response.ok) throw new Error(`npm lookup failed: HTTP ${response.status}`);+ const pkg = await response.json();+ assertMetadata(pkg, name, sha);+ if (!pkg.dist?.integrity || !pkg.dist?.tarball) throw new Error("Published preview has no immutable distribution pin.");+ return true;+}++export async function imageExists(sha, fetchImpl = fetch) {+ versionFor(sha);+ const tokenRes = await fetchImpl("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", { signal: AbortSignal.timeout(30_000) });+ if (!tokenRes.ok) throw new Error(`GHCR lookup failed: HTTP ${tokenRes.status}`);+ const { token } = await tokenRes.json();+ if (typeof token !== "string") throw new Error("GHCR did not return a pull token.");+ const base = "https://ghcr.io/v2/paperclipai/paperclip";+ const headers = { Authorization: `Bearer ${token}`, Accept: "application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" };+ const get = (url) => fetchImpl(url, { headers, redirect: "error", signal: AbortSignal.timeout(30_000) });+ let res = await get(`${base}/manifests/sha-${sha}-cloud`);+ if (res.status === 404) return false;+ if (!res.ok) throw new Error(`GHCR lookup failed: HTTP ${res.status}`);+ let manifest = await res.json();+ const digest = (value) => {+ if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value)) throw new Error("Invalid image digest.");+ return value;+ };+ if (Array.isArray(manifest.manifests)) {+ const amd64 = manifest.manifests.find((entry) => entry.platform?.os === "linux" && entry.platform?.architecture === "amd64");+ if (!amd64) throw new Error("Cloud image has no Linux amd64 manifest.");+ res = await get(`${base}/manifests/${digest(amd64.digest)}`);+ if (!res.ok) throw new Error(`GHCR manifest lookup failed: HTTP ${res.status}`);+ manifest = await res.json();+ }+ res = await fetchImpl(`${base}/blobs/${digest(manifest.config?.digest)}`, { headers, redirect: "manual", signal: AbortSignal.timeout(30_000) });+ // Registry blob storage may redirect to its signed storage URL. Follow only+ // with no Authorization header, so the GHCR token cannot leave the registry.+ if ([301, 302, 307, 308].includes(res.status)) {+ const location = new URL(res.headers.get("location"));+ if (location.protocol !== "https:" || location.username || location.password) throw new Error("Invalid registry blob redirect.");+ res = await fetchImpl(location.href, { redirect: "error", signal: AbortSignal.timeout(30_000) });+ }+ if (!res.ok) throw new Error(`GHCR config lookup failed: HTTP ${res.status}`);+ const config = await res.json();+ if (config.config?.Labels?.["org.opencontainers.image.revision"] !== sha) throw new Error("Existing SHA image tag does not match the requested full commit.");+ return true;+}++/** Publication loads image data, but never runs a container or source scripts. */+export async function publishImage(file, sha, { exec = execFileSync, fetchImpl = fetch } = {}) {+ versionFor(sha);+ const image = `ghcr.io/paperclipai/paperclip:sha-${sha}-cloud`;+ if (await imageExists(sha, fetchImpl)) { console.log("Reusing the verified SHA cloud image."); return; }+ exec("docker", ["load", "--input", path.resolve(file)], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 });+ const [metadata] = JSON.parse(exec("docker", ["image", "inspect", image], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }));+ if (metadata?.Config?.Labels?.["org.opencontainers.image.revision"] !== sha || metadata.Os !== "linux" || metadata.Architecture !== "amd64" ||+ !/^sha256:[0-9a-f]{64}$/.test(metadata.Id ?? "")) throw new Error("Built image identity or platform does not match the request.");+ // Push only this verified image ID under the one permitted tag, regardless+ // of any additional tag names present in the untrusted Docker archive.+ exec("docker", ["tag", metadata.Id, image], { stdio: "inherit" });+ exec("docker", ["push", image], { stdio: "inherit" });+}++export function packPreview(source, output, sha, { exec = execFileSync } = {}) {+ versionFor(sha);+ source = path.resolve(source); output = path.resolve(output);+ if (exec("git", ["rev-parse", "HEAD"], { cwd: source, encoding: "utf8" }).trim() !== sha) throw new Error("Source checkout differs from the requested commit.");+ mkdirSync(output, { recursive: true });+ for (const short of ["shared", "db"]) {+ exec("pnpm", ["--filter", `@paperclipai/${short}`, "build"], { cwd: source, stdio: "inherit" });+ const packageDir = path.join(source, "packages", short);+ const originalText = readFileSync(path.join(packageDir, "package.json"), "utf8");+ const original = JSON.parse(originalText);+ const pkg = previewManifest(original, sha);+ const staging = path.join(output, `package-${short}`);+ if ((pkg.bundleDependencies ?? []).length) {+ // The established helper materializes patched embedded-postgres instead+ // of publishing pnpm's dependency symlinks.+ writeFileSync(path.join(packageDir, "package.json"), JSON.stringify(pkg));+ try { prepareBundledPackage(packageDir, staging, { sourceRoot: source }); }+ finally { writeFileSync(path.join(packageDir, "package.json"), originalText); }+ } else {+ mkdirSync(staging, { recursive: true });+ cpSync(path.join(packageDir, "dist"), path.join(staging, "dist"), { recursive: true });+ writeFileSync(path.join(staging, "package.json"), JSON.stringify(pkg));+ }+ const packed = JSON.parse(exec("npx", ["--yes", "[email protected]", "pack", "--ignore-scripts", "--json", "--pack-destination", output], { cwd: staging, encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }));+ renameSync(path.join(output, path.basename(packed[0].filename)), path.join(output, `${short}.tgz`));+ assertMetadata(tarManifest(readFileSync(path.join(output, `${short}.tgz`))), `@paperclipai/${short}`, sha);+ }+}++export async function publishPreview(dir, sha, { fetchImpl = fetch, exec = execFileSync, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {+ for (const short of ["shared", "db"]) {+ const name = `@paperclipai/${short}`;+ const file = path.resolve(dir, `${short}.tgz`);+ const bytes = readFileSync(file);+ assertMetadata(tarManifest(bytes), name, sha);+ if (await packageExists(name, sha, fetchImpl)) { console.log(`Reusing ${name}@${versionFor(sha)}`); continue; }+ console.log(`Publishing ${name}@${versionFor(sha)} (${createHash("sha256").update(bytes).digest("hex").slice(0, 12)})`);+ // No package checkout, lifecycle scripts, npmrc, or branch code runs here.+ exec("npm", ["publish", file, "--tag", "preview", "--access", "public", "--ignore-scripts", "--provenance", "--registry", "https://registry.npmjs.org"], { stdio: "inherit" });+ let published = false;+ for (let attempt = 0; attempt < 60; attempt++) {+ if (await packageExists(name, sha, fetchImpl)) { published = true; break; }+ await sleep(10_000);+ }+ if (!published) throw new Error("npm accepted the preview but it is not yet visible. Retry reuses published packages.");+ }+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [command, ...args] = process.argv.slice(2);+ try {+ if (command === "plan") {+ const [sha, requestId, migrator] = args;+ validateRequest(sha, requestId);+ if (process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Preview workflow definitions must run from master.");+ const image = !await imageExists(sha);+ const packages = migrator === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha));+ appendFileSync(process.env.GITHUB_OUTPUT, `image=${image}\npackages=${packages}\n`);+ } else if (command === "pack") packPreview(...args);+ else if (command === "publish") await publishPreview(...args);+ else if (command === "publish-image") await publishImage(...args);+ else if (command === "result") {+ const [sha, requestId] = args;+ validateRequest(sha, requestId);+ if (!await imageExists(sha)) throw new Error("Cloud image is still missing.");+ if (process.env.PREVIEW_MIGRATOR === "true" && !(await packageExists("@paperclipai/shared", sha) && await packageExists("@paperclipai/db", sha))) throw new Error("Preview packages are still missing.");+ mkdirSync("stack-deploy-result", { recursive: true });+ writeFileSync("stack-deploy-result/result.json", JSON.stringify({ version: 1, stage: "build", requestId, sha, status: "ready" }) + "\n");+ } else throw new Error("Expected plan, pack, publish, publish-image, or result.");+ } catch (error) { console.error(error.message); process.exitCode = 1; }+}diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjsnew file mode 100644index 0000000000..746f246cf9--- /dev/null+++ b/scripts/preview-artifacts.test.mjs@@ -0,0 +1,128 @@+import test from "node:test";+import assert from "node:assert/strict";+import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import path from "node:path";+import { gzipSync } from "node:zlib";+import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs";++const sha = "a".repeat(40);+const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";+const manifest = (name) => previewManifest({ name, version: "0.0.0", dependencies: name.endsWith("/db") ? { "@paperclipai/shared": "workspace:*" } : {}, publishConfig: { exports: { ".": "./dist/index.js" } } }, sha);+function pack(pkg) {+ const b = Buffer.from(JSON.stringify(pkg)); const h = Buffer.alloc(512);+ h.write("package/package.json"); h.write(b.length.toString(8).padStart(11, "0"), 124, 11); h[156] = 48;+ const padded = Buffer.alloc(Math.ceil(b.length / 512) * 512); b.copy(padded);+ return gzipSync(Buffer.concat([h, padded, Buffer.alloc(1024)]));+}+const json = (body, status = 200) => new Response(JSON.stringify(body), { status });++test("preview request requires immutable SHA and correlation UUID", () => {+ validateRequest(sha, id);+ for (const ref of ["master", "origin/master", "a".repeat(7), "$(unsafe)", "A".repeat(40)]) assert.throws(() => versionFor(ref));+ assert.throws(() => validateRequest(sha, "not-a-request"));+});++test("preview manifests carry exact source, isolated versions and shared dependency", () => {+ const pkg = manifest("@paperclipai/db");+ assert.equal(pkg.version, `0.0.0-preview.g${sha}`);+ assert.equal(pkg.dependencies["@paperclipai/shared"], pkg.version);+ assert.deepEqual(pkg.exports, { ".": "./dist/index.js" });+ assertMetadata(pkg, "@paperclipai/db", sha);+ assert.throws(() => assertMetadata({ ...pkg, gitHead: "b".repeat(40) }, pkg.name, sha));+ assert.throws(() => assertMetadata({ ...pkg, dependencies: { "@paperclipai/shared": "latest" } }, pkg.name, sha));+ assert.deepEqual(tarManifest(pack(pkg)), pkg);+});++test("only 404 means an artifact is missing; auth and outages are fatal", async () => {+ assert.equal(await packageExists("@paperclipai/db", sha, async () => json({}, 404)), false);+ await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({}, 403)));+ await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({}, 503)));+ await assert.rejects(imageExists(sha, async () => json({}, 503)));+ assert.equal(await imageExists(sha, async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404)), false);+ await assert.rejects(packageExists("@paperclipai/db", sha, async () => json({ ...manifest("@paperclipai/db"), gitHead: "b".repeat(40) })));+});++test("publishing reuses existing previews and never executes package lifecycle hooks", async () => {+ const dir = mkdtempSync(path.join(tmpdir(), "preview-publish-test-"));+ const published = new Set(["@paperclipai/shared"]);+ const calls = [];+ try {+ for (const short of ["shared", "db"]) writeFileSync(path.join(dir, `${short}.tgz`), pack({ ...manifest(`@paperclipai/${short}`), scripts: { prepublishOnly: "do-not-run" } }));+ await publishPreview(dir, sha, {+ fetchImpl: async (url) => {+ const name = decodeURIComponent(new URL(url).pathname.split("/")[1]);+ return published.has(name) ? json({ ...manifest(name), dist: { integrity: "test-integrity", tarball: "https://registry.npmjs.org/package.tgz" } }) : json({}, 404);+ },+ exec: (command, args) => { calls.push({ command, args }); published.add("@paperclipai/db"); },+ sleep: async () => {},+ });+ assert.equal(calls.length, 1);+ assert.equal(calls[0].command, "npm");+ assert.ok(calls[0].args.includes("--ignore-scripts"));+ assert.equal(calls[0].args[calls[0].args.indexOf("--tag") + 1], "preview");+ assert.ok(!calls[0].args.includes("canary"));+ } finally { rmSync(dir, { recursive: true, force: true }); }+});++test("preview workflow separates branch compilation from trusted publishing", () => {+ const workflow = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8");+ const builder = workflow.split(" package_preview:")[1].split(" publish_preview:")[0];+ const publisher = workflow.split(" publish_preview:")[1].split(" image_preview:")[0];+ const image = workflow.split(" image_preview:")[1].split(" publish_image_preview:")[0];+ const imagePublisher = workflow.split(" publish_image_preview:")[1].split(" result_preview:")[0];+ assert.doesNotMatch(builder, /id-token: write|packages: write|secrets\./);+ assert.doesNotMatch(publisher, /ref: \$\{\{ inputs.source_ref|working-directory: source|pnpm install/);+ assert.match(publisher, /environment: npm-canary/);+ assert.match(image, /PAPERCLIP_BUILD_COMMIT=\$\{\{ inputs.source_ref \}\}/);+ assert.doesNotMatch(image, /cache-(?:to|from):|canary-cloud|latest-cloud|packages: write|secrets\./);+ assert.doesNotMatch(imagePublisher, /ref: \$\{\{ inputs.source_ref|docker\/build-push-action|pnpm install/);+ assert.match(imagePublisher, /publish-image/);+ assert.match(imagePublisher, /environment: npm-canary/);+ assert.match(imagePublisher, /github.ref == 'refs\/heads\/master'/);+ assert.match(publisher, /github.ref == 'refs\/heads\/master'/);+ assert.doesNotMatch(workflow.split(" verify_canary:")[0], /uses: [^\n]+@v\d/);+ assert.match(workflow, /Stack deploy \{0\} build/);+});+++test("existing image reuse verifies the full revision behind the immutable tag", async () => {+ const digest = "sha256:" + "b".repeat(64);+ for (const revision of [sha, "c".repeat(40)]) {+ const fetchImpl = async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) :+ url.includes("/blobs/") ? json({ config: { Labels: { "org.opencontainers.image.revision": revision } } }) :+ url.endsWith(digest) ? json({ config: { digest } }) : json({ manifests: [{ digest, platform: { os: "linux", architecture: "amd64" } }] });+ if (revision === sha) assert.equal(await imageExists(sha, fetchImpl), true);+ else await assert.rejects(imageExists(sha, fetchImpl), /full commit/);+ }+});+++test("image publisher verifies source and platform before pushing exactly one immutable tag", async () => {+ for (const revision of [sha, "c".repeat(40)]) {+ const calls = [];+ const operation = publishImage("preview-image.tar", sha, {+ fetchImpl: async (url) => url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404),+ exec: (command, args) => {+ calls.push({ command, args });+ if (args[0] === "image") return JSON.stringify([{ Id: "sha256:" + "b".repeat(64), Os: "linux", Architecture: "amd64", Config: { Labels: { "org.opencontainers.image.revision": revision } } }]);+ return "";+ },+ });+ if (revision === sha) {+ await operation;+ assert.deepEqual(calls.filter((call) => call.args[0] === "push").map((call) => call.args), [["push", `ghcr.io/paperclipai/paperclip:sha-${sha}-cloud`]]);+ } else { await assert.rejects(operation, /identity/); assert.ok(!calls.some((call) => call.args[0] === "push")); }+ assert.ok(!calls.some((call) => ["run", "build"].includes(call.args[0])));+ }+});+++test("commits sharing a short prefix use separate full-SHA image addresses", async () => {+ const urls = [];+ const fetchImpl = async (url) => { urls.push(url); return url.includes("/token?") ? json({ token: "test-pull-token" }) : json({}, 404); };+ const other = sha.slice(0, 7) + "b".repeat(33);+ await imageExists(sha, fetchImpl);+ await imageExists(other, fetchImpl);+ assert.deepEqual(urls.filter((url) => url.includes("/manifests/")), [sha, other].map((commit) => `https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${commit}-cloud`));+});