The Open Framework for Building Agents
fix(eve): accept only own-queue deliveries on the self-hosted workflow flow route (#5219)
Colton Paddenaaae448b737e
diff --git a/.changeset/local-world-delivery-auth.md b/.changeset/local-world-delivery-auth.mdnew file mode 100644index 000000000..9db416ef2--- /dev/null+++ b/.changeset/local-world-delivery-auth.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Self-hosted `eve start` servers now reject requests to `/.well-known/workflow/v1/flow` that did not come from the server's own workflow queue, so a client can no longer forge a workflow delivery with a chosen run context or principal. Startup always recovers pending and running local workflow runs through the authenticated queue, ignoring `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS`; queue deliveries must stay on the process that enqueued them.diff --git a/docs/guides/deployment/self-hosting.md b/docs/guides/deployment/self-hosting.mdindex e0880b00d..32617f11f 100644--- a/docs/guides/deployment/self-hosting.md+++ b/docs/guides/deployment/self-hosting.md@@ -28,6 +28,10 @@ Don’t rely on `vercelOidc()` as the only production authenticator outside VercThe default local Workflow world stores run state under `.eve/.workflow-data`. Mount that directory on persistent storage so runs survive process and container replacement.+With the default local world, each `eve start` process accepts workflow queue deliveries only from its own queue. Keep deliveries on the process that enqueued them. Do not point `WORKFLOW_LOCAL_BASE_URL` at a load balancer that can forward deliveries to another replica; that replica rejects them with HTTP 401. Use a custom world designed for shared queue delivery when scaling across replicas.++On startup, eve re-enqueues pending and running local workflow runs with a new per-process delivery proof. Local recovery always runs; `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS=0` or `false` does not disable it in `eve start`.+You can instead select an installed Workflow world package in the root `agent.ts`:```typescriptdiff --git a/packages/eve/src/internal/application/compiled-artifacts.test.ts b/packages/eve/src/internal/application/compiled-artifacts.test.tsindex b79991fa0..b60b7ca1c 100644--- a/packages/eve/src/internal/application/compiled-artifacts.test.ts+++ b/packages/eve/src/internal/application/compiled-artifacts.test.ts@@ -39,6 +39,8 @@ describe("createWorkflowWorldPluginSource", () => {expect(source).toContain("/compiled/@workflow/world-local/index.js");expect(source).toContain("resolveLocalWorkflowWorldDataDirectory(process.cwd())");expect(source).toContain("applyLocalWorkflowWorldDeliveryTimeoutDefaults");+ expect(source).toContain("authenticateLocalWorkflowDeliveries(");+ expect(source).toContain("recoverActiveRuns: false");expect(source.indexOf("applyLocalWorkflowWorldDeliveryTimeoutDefaults();")).toBeLessThan(source.indexOf("workflowWorldModule.createWorld"),);@@ -59,6 +61,7 @@ describe("createWorkflowWorldPluginSource", () => {expect(source).toContain("/compiled/@workflow/world-vercel/index.js");expect(source).toMatch(/headers: \{ "User-Agent": "eve\/.+" \}/);expect(source).toContain("applyVercelWorkflowWorldDefaults(workflowWorld);");+ expect(source).not.toContain("authenticateLocalWorkflowDeliveries");expect(source.indexOf('installWorkflowDeploymentId("')).toBeGreaterThan(-1);expect(source.indexOf('installWorkflowDeploymentId("')).toBeLessThan(source.indexOf("await workflowWorldModule.createWorld("),diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.tsindex cf29d939e..134f584a3 100644--- a/packages/eve/src/internal/application/compiled-artifacts.ts+++ b/packages/eve/src/internal/application/compiled-artifacts.ts@@ -328,17 +328,24 @@ function resolveWorkflowWorldWiring(packageName: string): WorkflowWorldWiring {const deliveryTimeoutsImportSpecifier = stringifyEsmImportSpecifier(resolvePackageSourceFilePath("src/internal/workflow/local-world-delivery-timeouts.ts"),);+ const deliveryAuthImportSpecifier = stringifyEsmImportSpecifier(+ resolvePackageSourceFilePath("src/internal/workflow/local-world-delivery-auth.ts"),+ );const moduleImportSpecifier = resolvePackageCompiledFilePath(`src/compiled/${packageName}/index.js`,);const importLines = `import { applyLocalWorkflowWorldDeliveryTimeoutDefaults } from ${deliveryTimeoutsImportSpecifier};+import { authenticateLocalWorkflowDeliveries } from ${deliveryAuthImportSpecifier};import { resolveLocalWorkflowWorldDataDirectory } from ${dataDirectoryImportSpecifier};`.trimStart();const createWorldSource = `applyLocalWorkflowWorldDeliveryTimeoutDefaults();-const workflowWorld = await workflowWorldModule.createWorld({- dataDir: resolveLocalWorkflowWorldDataDirectory(process.cwd()),-});`.trimStart();+const workflowWorld = authenticateLocalWorkflowDeliveries(+ await workflowWorldModule.createWorld({+ dataDir: resolveLocalWorkflowWorldDataDirectory(process.cwd()),+ recoverActiveRuns: false,+ }),+);`.trimStart();return {moduleImportSpecifier,extraImportLines: importLines.split("\n"),diff --git a/packages/eve/src/internal/workflow/local-world-delivery-auth.test.ts b/packages/eve/src/internal/workflow/local-world-delivery-auth.test.tsnew file mode 100644index 000000000..d44c7f1b7--- /dev/null+++ b/packages/eve/src/internal/workflow/local-world-delivery-auth.test.ts@@ -0,0 +1,136 @@+import { afterEach, describe, expect, it, vi } from "vitest";++import {+ authenticateLocalWorkflowDeliveries,+ LOCAL_WORKFLOW_DELIVERY_HEADER,+} from "#internal/workflow/local-world-delivery-auth.js";++const SECRET = "test-delivery-secret-0123456789";+const FLOW_URL = "http://localhost/.well-known/workflow/v1/flow";+const QUEUE_HEADERS = {+ "x-vqs-message-attempt": "1",+ "x-vqs-message-id": "msg_1",+ "x-vqs-queue-name": "__wkf_workflow_entry",+};++function createFakeWorld() {+ const enqueued: Array<{+ name: string;+ message: unknown;+ headers: Record<string, string> | undefined;+ }> = [];+ const start = vi.fn(async () => {});+ const world = {+ queue: async (+ name: string,+ message: unknown,+ options?: { headers?: Record<string, string> },+ ) => {+ enqueued.push({ name, message, headers: options?.headers });+ return { messageId: "msg_1" };+ },+ createQueueHandler:+ (_prefix: string, handler: (message: unknown) => Promise<void>) =>+ async (request: Request) => {+ await handler(await request.json());+ return Response.json({ ok: true });+ },+ runs: {+ list: vi.fn(async ({ pagination }: { pagination: { cursor?: string } }) =>+ pagination.cursor === undefined+ ? {+ cursor: "page_2",+ data: [{ runId: "wrun_1", workflowName: "entry" }],+ hasMore: true,+ }+ : {+ cursor: null,+ data: [{ runId: "wrun_2", workflowName: "other" }],+ hasMore: false,+ },+ ),+ },+ start,+ };+ const secured = authenticateLocalWorkflowDeliveries(world as never, SECRET);+ return { enqueued, secured, start, listRuns: world.runs.list };+}++describe("authenticateLocalWorkflowDeliveries", () => {+ afterEach(() => {+ vi.unstubAllEnvs();+ });++ it("stamps every enqueue and keeps the caller's headers", async () => {+ const { enqueued, secured } = createFakeWorld();+ await secured.queue("__wkf_workflow_entry" as never, { runId: "wrun_1" } as never, {+ headers: { "x-other": "1" },+ });+ expect(enqueued).toEqual([+ {+ name: "__wkf_workflow_entry",+ message: { runId: "wrun_1" },+ headers: { "x-other": "1", [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },+ },+ ]);+ });++ it("rejects a forged delivery before the handler sees its message", async () => {+ const { secured } = createFakeWorld();+ const inner = vi.fn(async () => {});+ const handle = secured.createQueueHandler("__wkf_workflow_" as never, inner as never);+ const body = JSON.stringify({+ runInput: {+ input: [+ {+ serializedContext: {+ "eve.auth": { principalId: "victim", principalType: "user" },+ },+ },+ ],+ },+ });+ const post = (headers: Record<string, string>) =>+ handle(new Request(FLOW_URL, { body, headers, method: "POST" }));++ expect((await post(QUEUE_HEADERS)).status).toBe(401);+ expect(+ (await post({ ...QUEUE_HEADERS, [LOCAL_WORKFLOW_DELIVERY_HEADER]: "guess" })).status,+ ).toBe(401);+ expect(inner).not.toHaveBeenCalled();++ const trusted = await post({ ...QUEUE_HEADERS, [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET });+ expect(trusted.status).toBe(200);+ expect(inner).toHaveBeenCalledTimes(1);+ });++ it("re-enqueues active runs through the stamped queue when the world starts", async () => {+ vi.stubEnv("WORKFLOW_QUEUE_NAMESPACE", "evetest");+ const { enqueued, secured, start, listRuns } = createFakeWorld();+ await secured.start?.();+ expect(start).toHaveBeenCalledTimes(1);+ expect(enqueued).toEqual([+ {+ name: "__evetest_wkf_workflow_entry",+ message: { runId: "wrun_1" },+ headers: { [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },+ },+ {+ name: "__evetest_wkf_workflow_other",+ message: { runId: "wrun_2" },+ headers: { [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },+ },+ ]);+ expect(listRuns.mock.calls).toEqual([+ [{ pagination: { cursor: undefined }, resolveData: "none", status: ["pending", "running"] }],+ [{ pagination: { cursor: "page_2" }, resolveData: "none", status: ["pending", "running"] }],+ ]);+ });++ it("rejects recovery without an installed namespace", async () => {+ vi.stubEnv("WORKFLOW_QUEUE_NAMESPACE", undefined);+ const { enqueued, secured } = createFakeWorld();+ await expect(secured.start?.()).rejects.toThrow("WORKFLOW_QUEUE_NAMESPACE");+ expect(enqueued).toEqual([]);+ });+});diff --git a/packages/eve/src/internal/workflow/local-world-delivery-auth.ts b/packages/eve/src/internal/workflow/local-world-delivery-auth.tsnew file mode 100644index 000000000..6aef407e8--- /dev/null+++ b/packages/eve/src/internal/workflow/local-world-delivery-auth.ts@@ -0,0 +1,83 @@+import { randomBytes } from "node:crypto";++import type { ValidQueueName } from "#compiled/@workflow/world/index.js";+import type { LocalWorld } from "#compiled/@workflow/world-local/index.js";+import { timingSafeEqualStrings } from "#internal/nitro/dev-client-address.js";+import {+ workflowQueuePrefix,+ WORKFLOW_QUEUE_NAMESPACE_ENV,+} from "#internal/workflow/queue-namespace.js";++export const LOCAL_WORKFLOW_DELIVERY_HEADER = "x-eve-workflow-delivery";++/**+ * Makes the local World's flow route accept only deliveries this process+ * enqueued. The route is as reachable as the server, and world-local trusts+ * whatever `x-vqs-*` headers and body arrive, so without a proof any client+ * could forge a delivery, including its run context and principal.+ *+ * The proof is a random per-process secret that never leaves the process:+ * the local World only delivers to the server that enqueued. world-local's+ * own startup recovery enqueues past the wrapped queue, so the World must be+ * created with `recoverActiveRuns: false`; `start()` recovers through the+ * stamped queue instead.+ *+ * Mutates and returns `world`.+ */+export function authenticateLocalWorkflowDeliveries(+ world: LocalWorld,+ secret: string = randomBytes(32).toString("hex"),+): LocalWorld {+ const enqueue = world.queue.bind(world);+ world.queue = async (queueName, message, options) =>+ await enqueue(queueName, message, {+ ...options,+ headers: { ...options?.headers, [LOCAL_WORKFLOW_DELIVERY_HEADER]: secret },+ });++ const createQueueHandler = world.createQueueHandler.bind(world);+ world.createQueueHandler = (queueNamePrefix, handler) => {+ const deliver = createQueueHandler(queueNamePrefix, handler);+ return async (request) => {+ const proof = request.headers.get(LOCAL_WORKFLOW_DELIVERY_HEADER);+ if (proof === null || !timingSafeEqualStrings(proof, secret)) {+ return Response.json({ error: "Untrusted workflow delivery." }, { status: 401 });+ }+ return await deliver(request);+ };+ };++ const start = world.start?.bind(world);+ world.start = async () => {+ await start?.();+ await reenqueueActiveRuns(world);+ };+ return world;+}++/** Mirrors world-local's startup recovery, but through the stamped `world.queue`. */+async function reenqueueActiveRuns(world: LocalWorld): Promise<void> {+ const namespace = process.env[WORKFLOW_QUEUE_NAMESPACE_ENV];+ if (!namespace) {+ throw new Error(+ `${WORKFLOW_QUEUE_NAMESPACE_ENV} must be installed before local workflow recovery.`,+ );+ }+ const prefix = workflowQueuePrefix(namespace);+ let cursor: string | undefined;+ do {+ const page = await world.runs.list({+ pagination: { cursor },+ resolveData: "none",+ status: ["pending", "running"],+ });+ for (const run of page.data) {+ try {+ await world.queue(`${prefix}${run.workflowName}` as ValidQueueName, { runId: run.runId });+ } catch (error) {+ console.warn(`[eve] Failed to re-enqueue workflow run ${run.runId}:`, error);+ }+ }+ cursor = page.hasMore ? (page.cursor ?? undefined) : undefined;+ } while (cursor !== undefined);+}diff --git a/packages/eve/src/internal/workflow/queue-namespace.ts b/packages/eve/src/internal/workflow/queue-namespace.tsindex 23bf6b6e5..7a23bedc3 100644--- a/packages/eve/src/internal/workflow/queue-namespace.ts+++ b/packages/eve/src/internal/workflow/queue-namespace.ts@@ -9,9 +9,14 @@ export function deriveEveWorkflowQueueNamespace(agentName: string): string {return `eve${encodedAgentName}`;}+/** Formats the Workflow queue prefix for an installed namespace. */+export function workflowQueuePrefix(namespace: string): string {+ return `__${namespace}_wkf_workflow_`;+}+/** Derives the queue prefix consumed by an eve agent's workflow handler. */export function deriveEveWorkflowQueuePrefix(agentName: string): string {- return `__${deriveEveWorkflowQueueNamespace(agentName)}_wkf_workflow_`;+ return workflowQueuePrefix(deriveEveWorkflowQueueNamespace(agentName));}/** Derives the queue topic registered for an eve agent's workflow handler. */diff --git a/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.ts b/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.tsnew file mode 100644index 000000000..5b98f8eba--- /dev/null+++ b/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.ts@@ -0,0 +1,59 @@+import { afterEach, expect, it, vi } from "vitest";++import { Client } from "../../src/client/client.js";+import { buildApplication } from "../../src/internal/nitro/host.js";+import { useScenarioApp } from "../../src/internal/testing/scenario-app.js";+import { startPackagedEveStart } from "./eve-start-harness.js";++const scenarioApp = useScenarioApp();++afterEach(() => vi.unstubAllEnvs());++it("accepts only own-queue workflow deliveries in the built production server", async () => {+ vi.stubEnv("EVE_MOCK_AUTHORED_MODELS", "");+ vi.stubEnv("NODE_ENV", "production");+ const { appRoot } = await scenarioApp({+ name: "local-workflow-delivery-auth",+ installDependencies: true,+ files: {+ "agent/agent.ts": `import { defineAgent } from "eve";+import { mockModel } from "eve/evals";+export default defineAgent({+ model: mockModel(() => "Noted, Alice."),+ modelContextWindowTokens: 32_000,+});`,+ "agent/instructions.md": "Acknowledge Alice's messages briefly.\n",+ "agent/channels/eve.ts": `import { httpBasic } from "eve/channels/auth";+import { eveChannel } from "eve/channels/eve";+export default eveChannel({ auth: httpBasic({ username: "alice", password: "fixture" }) });`,+ },+ });+ await buildApplication(appRoot, { skipSandboxPrewarm: true });+ const server = await startPackagedEveStart(appRoot);+ try {+ for (const proof of [undefined, "incorrect-delivery-proof"]) {+ const response = await fetch(new URL("/.well-known/workflow/v1/flow", server.url), {+ method: "POST",+ body: "{}",+ headers: proof === undefined ? {} : { "x-eve-workflow-delivery": proof },+ signal: AbortSignal.timeout(10_000),+ });+ expect(response.status, server.stderr()).toBe(401);+ expect(await response.json()).toEqual({ error: "Untrusted workflow delivery." });+ }++ const client = new Client({+ host: server.url,+ auth: { basic: { username: "alice", password: "fixture" } },+ });+ const { session } = await client.sessions.create();+ const result = await (+ await session.send("Alice asks you to acknowledge receipt.", {+ signal: AbortSignal.timeout(30_000),+ })+ ).result();+ expect(result.message, server.stderr()).toBe("Noted, Alice.");+ } finally {+ await server.stop();+ }+}, 120_000);