Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

refactor(eve): proxy private package artifacts (#2004)

OwenKephart08d30b9bf5ce
diff --git a/apps/package-artifacts/README.md b/apps/package-artifacts/README.md
index 5baeb01ef..ab9f04a48 100644
--- a/apps/package-artifacts/README.md
+++ b/apps/package-artifacts/README.md
@@ -1,6 +1,6 @@
# eve package artifacts
-Git-linked Vercel project that builds an eve tarball from each `vercel/eve` `main` commit and uploads immutable SHA-addressed package and manifest artifacts to public Vercel Blob using deployment OIDC.
+Git-linked Vercel project that builds an eve tarball from each `vercel/eve` `main` commit and uploads immutable SHA-addressed package and manifest artifacts to private Vercel Blob using deployment OIDC.
```text
/main/eve.tgz
@@ -8,20 +8,20 @@ Git-linked Vercel project that builds an eve tarball from each `vercel/eve` `mai
/<full-sha>/eve.tgz
```
-Set `EVE_PACKAGE_BASE_URL` in Production to the project's public HTTPS origin, without a route suffix. For example, if it is `https://packages.example.com`, initialize an agent from the current `main` build with:
+The publisher uses Vercel's `VERCEL_PROJECT_PRODUCTION_URL` system environment variable as the public package domain. For example, if the production domain is `packages.example.com`, initialize an agent from the current `main` build with:
```bash
npm exec --yes --package=https://packages.example.com/main/eve.tgz -- eve init my-agent
```
-The production deployment resolves `main` to its checked-out commit. Its `latest.json` exposes metadata for that package, including the source SHA, immutable Blob URL, and checksum. The packaged CLI stamps its immutable commit URL into generated projects, so a project created through the moving `main` URL remains pinned to the package used to create it.
+The production deployment resolves `main` to its checked-out commit. Its `latest.json` exposes metadata for that package, including the source SHA, immutable package URL, and checksum. The packaged CLI stamps its immutable commit URL into generated projects, so a project created through the moving `main` URL remains pinned to the package used to create it. The package route reads private Blob objects with deployment OIDC and streams them through the public project domain.
The Vercel project must:
- use this directory as its project root;
+- enable access to Vercel system environment variables;
- deploy `main` to Production;
-- connect a public Blob store to Production;
-- set `EVE_PACKAGE_BASE_URL` to its public production origin;
+- connect a private Blob store to Production;
- omit `BLOB_READ_WRITE_TOKEN` so Blob writes use Vercel OIDC; and
- disable Deployment Protection so npm can reach the production origin anonymously.
diff --git a/apps/package-artifacts/api/package.js b/apps/package-artifacts/api/package.js
index dbbd0befe..48ed1cbae 100644
--- a/apps/package-artifacts/api/package.js
+++ b/apps/package-artifacts/api/package.js
@@ -1,6 +1,9 @@
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+
import { get } from "@vercel/blob";
-import { SHA_PATTERN, packageManifestPath } from "../lib/package.mjs";
+import { SHA_PATTERN, packageArtifactPath, packageManifestPath } from "../lib/package.mjs";
export default async function handler(request, response) {
const ref = request.query.ref;
@@ -17,15 +20,23 @@ export default async function handler(request, response) {
const manifest = await resolveManifest(sourceSha);
if (manifest === undefined) return packageNotFound(response);
- response.setHeader(
- "Cache-Control",
- ref === "main" ? "public, max-age=60" : "public, max-age=31536000, immutable",
- );
if (request.query.manifest === "1") {
if (ref !== "main") return packageNotFound(response);
+ response.setHeader("Cache-Control", "public, max-age=60");
return response.status(200).json(manifest);
}
- return response.redirect(302, manifest.tarball);
+ if (ref === "main") {
+ response.setHeader("Cache-Control", "public, max-age=60");
+ return response.redirect(302, manifest.tarball);
+ }
+
+ const artifact = await get(packageArtifactPath(sourceSha), { access: "private" });
+ if (artifact === null) return packageNotFound(response);
+
+ response.setHeader("Cache-Control", "public, max-age=31536000, immutable");
+ response.setHeader("Content-Type", "application/gzip");
+ await pipeline(Readable.fromWeb(artifact.stream), response);
+ return undefined;
}
function packageNotFound(response) {
@@ -33,7 +44,7 @@ function packageNotFound(response) {
}
async function resolveManifest(sourceSha) {
- const result = await get(packageManifestPath(sourceSha), { access: "public" });
+ const result = await get(packageManifestPath(sourceSha), { access: "private" });
if (result === null) return undefined;
const manifest = JSON.parse(await new Response(result.stream).text());
diff --git a/apps/package-artifacts/api/package.test.mjs b/apps/package-artifacts/api/package.test.mjs
index d1bdf65d2..cc3b51f5a 100644
--- a/apps/package-artifacts/api/package.test.mjs
+++ b/apps/package-artifacts/api/package.test.mjs
@@ -1,3 +1,5 @@
+import { PassThrough } from "node:stream";
+
import { beforeEach, describe, expect, test, vi } from "vitest";
const get = vi.fn();
@@ -8,25 +10,27 @@ const sha = "a".repeat(40);
const manifest = {
sourceSha: sha,
version: `0.33.0+main.${sha}`,
- tarball: "https://example.public.blob.vercel-storage.com/eve.tgz",
+ tarball: `https://packages.example.com/${sha}/eve.tgz`,
sha256: "b".repeat(64),
};
function response() {
- return {
- status: vi.fn().mockReturnThis(),
- send: vi.fn().mockReturnThis(),
- json: vi.fn().mockReturnThis(),
- redirect: vi.fn().mockReturnThis(),
- setHeader: vi.fn(),
- };
+ const stream = new PassThrough();
+ stream.status = vi.fn().mockReturnValue(stream);
+ stream.send = vi.fn().mockReturnValue(stream);
+ stream.json = vi.fn().mockReturnValue(stream);
+ stream.redirect = vi.fn().mockReturnValue(stream);
+ stream.setHeader = vi.fn();
+ return stream;
}
beforeEach(() => {
vi.clearAllMocks();
process.env.VERCEL_GIT_COMMIT_SHA = sha;
- get.mockImplementation(async () => ({
- stream: new Blob([JSON.stringify(manifest)]).stream(),
+ get.mockImplementation(async (pathname) => ({
+ stream: new Blob([
+ pathname.endsWith("manifest.json") ? JSON.stringify(manifest) : "package bytes",
+ ]).stream(),
}));
});
@@ -36,7 +40,7 @@ describe("package route", () => {
await handler({ query: { ref: "main" } }, res);
expect(get).toHaveBeenCalledWith(`packages/${sha}/manifest.json`, {
- access: "public",
+ access: "private",
});
expect(res.setHeader).toHaveBeenCalledWith("Cache-Control", "public, max-age=60");
expect(res.redirect).toHaveBeenCalledWith(302, manifest.tarball);
@@ -53,11 +57,15 @@ describe("package route", () => {
test("serves SHA tarballs but not SHA latest manifests", async () => {
const artifact = response();
await handler({ query: { ref: sha } }, artifact);
+ expect(get).toHaveBeenLastCalledWith(`packages/${sha}/eve.tgz`, {
+ access: "private",
+ });
expect(artifact.setHeader).toHaveBeenCalledWith(
"Cache-Control",
"public, max-age=31536000, immutable",
);
- expect(artifact.redirect).toHaveBeenCalledWith(302, manifest.tarball);
+ expect(artifact.setHeader).toHaveBeenCalledWith("Content-Type", "application/gzip");
+ expect(artifact.read()).toEqual(Buffer.from("package bytes"));
const latest = response();
await handler({ query: { ref: sha, manifest: "1" } }, latest);
diff --git a/apps/package-artifacts/lib/package.mjs b/apps/package-artifacts/lib/package.mjs
index f040b24f5..bd0a85cdf 100644
--- a/apps/package-artifacts/lib/package.mjs
+++ b/apps/package-artifacts/lib/package.mjs
@@ -10,7 +10,7 @@ export function packageManifestPath(sourceSha) {
export function packageDependencyUrl(baseUrl, sourceSha) {
const url = new URL(baseUrl);
- if (url.protocol !== "https:") throw new Error("EVE_PACKAGE_BASE_URL must use HTTPS.");
+ if (url.protocol !== "https:") throw new Error("Package base URL must use HTTPS.");
url.pathname = `${url.pathname.replace(/\/$/, "")}/${sourceSha}/eve.tgz`;
return url.toString();
}
diff --git a/apps/package-artifacts/lib/package.test.mjs b/apps/package-artifacts/lib/package.test.mjs
index 5249c2434..0af6a93c0 100644
--- a/apps/package-artifacts/lib/package.test.mjs
+++ b/apps/package-artifacts/lib/package.test.mjs
@@ -24,7 +24,9 @@ describe("package artifacts", () => {
});
test("requires an HTTPS package base URL", () => {
- expect(() => packageDependencyUrl("http://packages.example.com", sha)).toThrow("use HTTPS");
+ expect(() => packageDependencyUrl("http://packages.example.com", sha)).toThrow(
+ "must use HTTPS",
+ );
});
test("prepares package metadata without mutating the source", () => {
diff --git a/apps/package-artifacts/scripts/build.mjs b/apps/package-artifacts/scripts/build.mjs
index 29f5d18e1..fffb08c36 100644
--- a/apps/package-artifacts/scripts/build.mjs
+++ b/apps/package-artifacts/scripts/build.mjs
@@ -3,7 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
-import { get, head, put } from "@vercel/blob";
+import { get, put } from "@vercel/blob";
import {
SHA_PATTERN,
@@ -20,7 +20,7 @@ const packageJsonPath = join(packageRoot, "package.json");
const artifactDirectory = join(appRoot, ".artifacts");
const sourceSha = process.env.VERCEL_GIT_COMMIT_SHA;
const branch = process.env.VERCEL_GIT_COMMIT_REF;
-const baseUrl = process.env.EVE_PACKAGE_BASE_URL;
+const productionDomain = process.env.VERCEL_PROJECT_PRODUCTION_URL;
if (!SHA_PATTERN.test(sourceSha ?? "")) {
throw new Error("VERCEL_GIT_COMMIT_SHA must be a 40-character Git commit SHA.");
@@ -34,15 +34,15 @@ if (branch !== "main" || process.env.VERCEL_ENV !== "production") {
process.exit(0);
}
-if (typeof baseUrl !== "string" || baseUrl.length === 0) {
- throw new Error("EVE_PACKAGE_BASE_URL is required for package publishing.");
+if (typeof productionDomain !== "string" || productionDomain.length === 0) {
+ throw new Error("VERCEL_PROJECT_PRODUCTION_URL is required for package publishing.");
}
const originalPackageJson = await readFile(packageJsonPath, "utf8");
const preparedPackageJson = preparePackageJson(JSON.parse(originalPackageJson), sourceSha);
const version = preparedPackageJson.version;
// Generated projects pin this deployment's immutable SHA route, never the moving main route.
-const dependencyUrl = packageDependencyUrl(baseUrl, sourceSha);
+const dependencyUrl = packageDependencyUrl(`https://${productionDomain}`, sourceSha);
const artifactPath = packageArtifactPath(sourceSha);
try {
@@ -53,10 +53,9 @@ try {
EVE_MAIN_DEPENDENCY_URL: dependencyUrl,
});
const sha256 = createHash("sha256").update(tarball).digest("hex");
- let artifact;
try {
- artifact = await put(artifactPath, tarball, {
- access: "public",
+ await put(artifactPath, tarball, {
+ access: "private",
addRandomSuffix: false,
allowOverwrite: false,
cacheControlMaxAge: 31_536_000,
@@ -65,25 +64,24 @@ try {
} catch (error) {
// Redeploys are valid only when this commit still produces identical package bytes.
if (!(error instanceof Error) || !error.message.includes("already exists")) throw error;
- artifact = await head(artifactPath);
- const publishedTarball = await fetch(artifact.url);
- if (!publishedTarball.ok) {
- throw new Error(`Published package artifact returned ${publishedTarball.status}.`);
+ const publishedArtifact = await get(artifactPath, { access: "private", useCache: false });
+ if (publishedArtifact === null) {
+ throw new Error("Published package artifact could not be read.");
}
const publishedSha256 = createHash("sha256")
- .update(Buffer.from(await publishedTarball.arrayBuffer()))
+ .update(Buffer.from(await new Response(publishedArtifact.stream).arrayBuffer()))
.digest("hex");
if (publishedSha256 !== sha256) {
throw new Error(`Commit ${sourceSha} was already published with different package contents.`);
}
}
- const manifest = { sourceSha, version, tarball: artifact.url, sha256 };
+ const manifest = { sourceSha, version, tarball: dependencyUrl, sha256 };
const manifestPath = packageManifestPath(sourceSha);
- const existingManifest = await get(manifestPath, { access: "public" });
+ const existingManifest = await get(manifestPath, { access: "private", useCache: false });
if (existingManifest === null) {
await put(manifestPath, JSON.stringify(manifest), {
- access: "public",
+ access: "private",
addRandomSuffix: false,
allowOverwrite: false,
cacheControlMaxAge: 31_536_000,
diff --git a/apps/package-artifacts/scripts/check-smoke.mjs b/apps/package-artifacts/scripts/check-smoke.mjs
index 83554c19b..0f7fc20f8 100644
--- a/apps/package-artifacts/scripts/check-smoke.mjs
+++ b/apps/package-artifacts/scripts/check-smoke.mjs
@@ -12,6 +12,10 @@ if (
}
if (artifact.status !== 302) throw new Error(`/main/eve.tgz returned ${artifact.status}.`);
+if (!location.startsWith(`${origin.replace(/\/$/, "")}/`)) {
+ throw new Error("Package route redirected outside the package deployment.");
+}
+
const tarball = await fetch(location);
if (!tarball.ok) throw new Error(`Package artifact returned ${tarball.status}.`);
const signature = new Uint8Array(await tarball.arrayBuffer()).subarray(0, 2);