Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(evals): verify remote deployments under the expected Vercel project (#4427)

Philipp Comans64301d168cea
diff --git a/.changeset/quiet-teams-verify.md b/.changeset/quiet-teams-verify.md
new file mode 100644
index 000000000..9e392f8a6
--- /dev/null
+++ b/.changeset/quiet-teams-verify.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Remote evals now resolve Vercel deployments using the owner and project from the environment or local project link, allowing protected deployments owned by a different team than the CLI's current team. Ambient credentials are withheld when the deployment does not match the expected project.
diff --git a/docs/evals/targets.mdx b/docs/evals/targets.mdx
index 68d8d494c..f38a2f312 100644
--- a/docs/evals/targets.mdx
+++ b/docs/evals/targets.mdx
@@ -36,7 +36,11 @@ These helpers handle the session protocol, retries, and event collection for the
## Authentication
-Local targets send no auth: `eve eval` owns the dev server it boots. For a remote `--url`, eve gets the expected Vercel owner and project from `VERCEL_ORG_ID` and `VERCEL_PROJECT_ID` when both are set. Otherwise it reads `.vercel/project.json`. Eve then asks Vercel to resolve the exact HTTPS origin and sends ambient credentials only when the project IDs match. An arbitrary URL remains anonymous.
+Local targets send no auth: `eve eval` owns the dev server it boots.
+
+For a remote `--url`, eve gets the expected Vercel owner and project from `VERCEL_ORG_ID` and `VERCEL_PROJECT_ID` when both values are nonblank. Otherwise it reads the local Vercel project link: `.vercel/project.json`, or the matching project in `.vercel/repo.json` for a linked monorepo.
+
+With a project reference, eve looks up the exact HTTPS origin under that owner, regardless of the Vercel CLI's current team. The deployment's project ID must match before eve sends ambient credentials. Without a project reference, eve verifies the URL under the CLI's current scope. URLs that Vercel cannot verify remain anonymous.
After verification, eve sends the available Vercel credentials:
diff --git a/packages/eve/src/evals/cli/eval-client.test.ts b/packages/eve/src/evals/cli/eval-client.test.ts
index e84873caa..75c341111 100644
--- a/packages/eve/src/evals/cli/eval-client.test.ts
+++ b/packages/eve/src/evals/cli/eval-client.test.ts
@@ -1,10 +1,21 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { readFile } from "node:fs/promises";
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { resolveTestVercelTarget } from "#internal/testing/verified-vercel-target.js";
import type { DevelopmentOidcTokenResolution } from "#services/dev-client/request-headers.js";
+import {
+ resolveVercelDeployment,
+ type VercelDeploymentResolutionDeps,
+} from "#setup/vercel-deployment.js";
import { createEvalClient, resolveEvalClientOptions } from "./eval-client.js";
+vi.mock("node:fs/promises", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("node:fs/promises")>()),
+ readFile: vi.fn(),
+}));
+
const VERIFIED_TARGET = await resolveTestVercelTarget({
host: "example.vercel.app",
projectId: "prj_example",
@@ -12,7 +23,12 @@ const VERIFIED_TARGET = await resolveTestVercelTarget({
});
describe("resolveEvalClientOptions", () => {
+ beforeEach(() => {
+ vi.mocked(readFile).mockRejectedValue(new Error("ENOENT"));
+ });
+
afterEach(() => {
+ vi.mocked(readFile).mockReset();
vi.restoreAllMocks();
vi.unstubAllEnvs();
});
@@ -130,3 +146,133 @@ describe("resolveEvalClientOptions", () => {
expect(headers.get("x-vercel-trusted-oidc-idp-token")).toBeNull();
});
});
+
+describe("remote eval project verification", () => {
+ const captureVercel = vi.fn<VercelDeploymentResolutionDeps["captureVercel"]>();
+ const resolveDevelopmentOidcToken = vi.fn(async () => ({
+ kind: "resolved" as const,
+ token: "ambient-token",
+ }));
+ const deployment = {
+ ownerId: "team_b",
+ projectId: "prj_target",
+ name: "target",
+ target: "preview",
+ };
+
+ beforeEach(() => {
+ vi.stubEnv("EVE_EVAL_AUTH_TOKEN", "");
+ vi.stubEnv("VERCEL_ORG_ID", "");
+ vi.stubEnv("VERCEL_PROJECT_ID", "");
+ vi.stubEnv("VERCEL_AUTOMATION_BYPASS_SECRET", "bypass-secret");
+ vi.mocked(readFile).mockResolvedValue(
+ JSON.stringify({ orgId: "team_b", projectId: "prj_target" }),
+ );
+ captureVercel.mockResolvedValue({ ok: true, stdout: JSON.stringify(deployment) });
+ vi.spyOn(globalThis, "fetch").mockResolvedValue(
+ Response.json({ ok: true, status: "ready", workflowId: "wf" }),
+ );
+ });
+
+ afterEach(() => {
+ captureVercel.mockReset();
+ resolveDevelopmentOidcToken.mockClear();
+ vi.mocked(readFile).mockReset();
+ vi.restoreAllMocks();
+ vi.unstubAllEnvs();
+ });
+
+ function createRemoteClient() {
+ return createEvalClient(
+ { kind: "remote", url: "https://target.vercel.app" },
+ {
+ workspaceRoot: "/workspace",
+ deps: {
+ resolveVercelDeployment: (input) =>
+ resolveVercelDeployment({ ...input, deps: { captureVercel } }),
+ resolveDevelopmentOidcToken,
+ },
+ },
+ );
+ }
+
+ it.each([
+ { source: "the complete environment pair", orgId: " team_b ", projectId: " prj_target " },
+ { source: "the project link", orgId: "", projectId: "" },
+ {
+ source: "the project link with an incomplete environment pair",
+ orgId: "team_a",
+ projectId: "",
+ },
+ {
+ source: "the project link with a blank environment value",
+ orgId: " ",
+ projectId: "prj_other",
+ },
+ ])("authenticates across CLI teams using $source", async ({ orgId, projectId }) => {
+ vi.stubEnv("VERCEL_ORG_ID", orgId);
+ vi.stubEnv("VERCEL_PROJECT_ID", projectId);
+ if (orgId.trim() && projectId.trim()) {
+ vi.mocked(readFile).mockResolvedValue(
+ JSON.stringify({ orgId: "team_a", projectId: "prj_other" }),
+ );
+ }
+ captureVercel.mockImplementation(async (args) => {
+ if (args[args.indexOf("--scope") + 1] !== "team_b") {
+ return {
+ ok: false,
+ failure: {
+ code: 1,
+ stdout: "",
+ stderr: "Error: Deployment not found (404)",
+ message: "Deployment lookup failed.",
+ },
+ };
+ }
+ return { ok: true, stdout: JSON.stringify(deployment) };
+ });
+
+ const client = await createRemoteClient();
+ await client.health();
+
+ const request = vi.mocked(fetch).mock.calls[0]?.[1];
+ const headers = new Headers(request?.headers);
+ expect(headers.get("authorization")).toBe("Bearer ambient-token");
+ expect(headers.get("x-vercel-trusted-oidc-idp-token")).toBe("ambient-token");
+ expect(headers.get("x-vercel-protection-bypass")).toBe("bypass-secret");
+ expect(request?.redirect).toBe("manual");
+ expect(resolveDevelopmentOidcToken).toHaveBeenCalledWith({
+ ownerId: "team_b",
+ projectId: "prj_target",
+ });
+ });
+
+ it("withholds ambient credentials when the deployment belongs to another linked project", async () => {
+ vi.mocked(readFile).mockResolvedValue(
+ JSON.stringify({ orgId: "team_b", projectId: "prj_other" }),
+ );
+
+ const client = await createRemoteClient();
+ await client.health();
+
+ const headers = new Headers(vi.mocked(fetch).mock.calls[0]?.[1]?.headers);
+ expect(headers.get("authorization")).toBeNull();
+ expect(headers.get("x-vercel-trusted-oidc-idp-token")).toBeNull();
+ expect(headers.get("x-vercel-protection-bypass")).toBeNull();
+ expect(resolveDevelopmentOidcToken).not.toHaveBeenCalled();
+ });
+
+ it("uses an explicit bearer without resolving a Vercel project", async () => {
+ vi.stubEnv("EVE_EVAL_AUTH_TOKEN", "static-token");
+
+ const client = await createRemoteClient();
+ await client.health();
+
+ const headers = new Headers(vi.mocked(fetch).mock.calls[0]?.[1]?.headers);
+ expect(headers.get("authorization")).toBe("Bearer static-token");
+ expect(headers.get("x-vercel-trusted-oidc-idp-token")).toBeNull();
+ expect(vi.mocked(readFile)).not.toHaveBeenCalled();
+ expect(captureVercel).not.toHaveBeenCalled();
+ expect(resolveDevelopmentOidcToken).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/eve/src/evals/cli/eval-client.ts b/packages/eve/src/evals/cli/eval-client.ts
index 565794fe7..7c2d3d086 100644
--- a/packages/eve/src/evals/cli/eval-client.ts
+++ b/packages/eve/src/evals/cli/eval-client.ts
@@ -1,5 +1,6 @@
import { Client } from "#client/client.js";
import type { ClientOptions } from "#client/types.js";
+import { readVercelProjectLink } from "#internal/vercel/project-link.js";
import { resolveDevelopmentClientOptions } from "#services/dev-client/client-options.js";
import {
resolveVerifiedRemoteDevelopmentClient,
@@ -46,7 +47,13 @@ export async function createEvalClient(
return new Client(base);
}
+ const orgId = process.env.VERCEL_ORG_ID?.trim();
+ const projectId = process.env.VERCEL_PROJECT_ID?.trim();
+ const projectSource =
+ orgId && projectId ? { orgId, projectId } : await readVercelProjectLink(options.workspaceRoot);
+
const { options: verified } = await resolveVerifiedRemoteDevelopmentClient({
+ projectSource,
serverUrl: target.url,
workspaceRoot: options.workspaceRoot,
deps: options.deps,
diff --git a/packages/eve/src/evals/mock-model.test.ts b/packages/eve/src/evals/mock-model.test.ts
index cfe213b3c..31f30a33f 100644
--- a/packages/eve/src/evals/mock-model.test.ts
+++ b/packages/eve/src/evals/mock-model.test.ts
@@ -149,6 +149,47 @@ describe("mockModel", () => {
]);
});
+ it("preserves eve file content parts in streamed mock requests", async () => {
+ const requests: MockModelRequest[] = [];
+ const content = [
+ { type: "text" as const, text: "Rendered stripes:" },
+ {
+ type: "file" as const,
+ data: { type: "data" as const, data: "iVBORw0KGgo=" },
+ filename: "stripes.png",
+ mediaType: "image/png",
+ },
+ ];
+ const result = streamText({
+ messages: [
+ {
+ role: "assistant",
+ content: [
+ { type: "tool-call", toolCallId: "render-1", toolName: "render-stripes", input: {} },
+ ],
+ },
+ {
+ role: "tool",
+ content: [
+ {
+ type: "tool-result",
+ toolCallId: "render-1",
+ toolName: "render-stripes",
+ output: { type: "content", value: content },
+ },
+ ],
+ },
+ ],
+ model: mockModel((request) => {
+ requests.push(request);
+ return "Received the image.";
+ }),
+ });
+
+ await expect(result.text).resolves.toBe("Received the image.");
+ expect(requests[0]!.toolResults[0]!.output).toEqual(content);
+ });
+
it("gives responders tool-result files in the tagged file shape", async () => {
const png = Buffer.from("\x89PNG\r\n\x1a\n", "latin1").toString("base64");
const requests: MockModelRequest[] = [];
diff --git a/packages/eve/src/setup/verified-remote-client.ts b/packages/eve/src/setup/verified-remote-client.ts
index b8fd0e3e5..bbbc892d9 100644
--- a/packages/eve/src/setup/verified-remote-client.ts
+++ b/packages/eve/src/setup/verified-remote-client.ts
@@ -8,6 +8,7 @@ import {
type DevelopmentOidcTokenFailure,
resolveDevelopmentOidcToken,
} from "#services/dev-client/request-headers.js";
+import type { VercelProjectReference } from "#setup/project-resolution.js";
import { resolveVercelDeployment, type VercelDeploymentResolution } from "./vercel-deployment.js";
@@ -38,6 +39,8 @@ export async function resolveVerifiedRemoteDevelopmentClient(input: {
readonly headers?: Readonly<Record<string, string>>;
readonly serverUrl: string;
readonly signal?: AbortSignal;
+ /** Expected project identity, used to scope and cross-check the deployment lookup. */
+ readonly projectSource?: VercelProjectReference;
readonly vercelScope?: string;
readonly workspaceRoot: string;
readonly deps?: Partial<VerifiedRemoteDevelopmentClientDeps>;
@@ -50,7 +53,9 @@ export async function resolveVerifiedRemoteDevelopmentClient(input: {
deploymentResolution = await deps.resolveVercelDeployment({
workspaceRoot: input.workspaceRoot,
host: new URL(input.serverUrl).host,
- scope: input.vercelScope,
+ ...(input.projectSource === undefined
+ ? { scope: input.vercelScope }
+ : { source: input.projectSource }),
signal: input.signal,
});