Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): accept only own-queue deliveries on the self-hosted workflow flow route (#5219)

Colton Paddenaaae448b737e
diff --git a/.changeset/local-world-delivery-auth.md b/.changeset/local-world-delivery-auth.md
new file mode 100644
index 000000000..9db416ef2
--- /dev/null
+++ b/.changeset/local-world-delivery-auth.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Self-hosted `eve start` servers now reject requests to `/.well-known/workflow/v1/flow` that did not come from the server's own workflow queue, so a client can no longer forge a workflow delivery with a chosen run context or principal. Startup always recovers pending and running local workflow runs through the authenticated queue, ignoring `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS`; queue deliveries must stay on the process that enqueued them.
diff --git a/docs/guides/deployment/self-hosting.md b/docs/guides/deployment/self-hosting.md
index e0880b00d..32617f11f 100644
--- a/docs/guides/deployment/self-hosting.md
+++ b/docs/guides/deployment/self-hosting.md
@@ -28,6 +28,10 @@ Don’t rely on `vercelOidc()` as the only production authenticator outside Verc
The default local Workflow world stores run state under `.eve/.workflow-data`. Mount that directory on persistent storage so runs survive process and container replacement.
+With the default local world, each `eve start` process accepts workflow queue deliveries only from its own queue. Keep deliveries on the process that enqueued them. Do not point `WORKFLOW_LOCAL_BASE_URL` at a load balancer that can forward deliveries to another replica; that replica rejects them with HTTP 401. Use a custom world designed for shared queue delivery when scaling across replicas.
+
+On startup, eve re-enqueues pending and running local workflow runs with a new per-process delivery proof. Local recovery always runs; `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS=0` or `false` does not disable it in `eve start`.
+
You can instead select an installed Workflow world package in the root `agent.ts`:
```typescript
diff --git a/packages/eve/src/internal/application/compiled-artifacts.test.ts b/packages/eve/src/internal/application/compiled-artifacts.test.ts
index b79991fa0..b60b7ca1c 100644
--- a/packages/eve/src/internal/application/compiled-artifacts.test.ts
+++ b/packages/eve/src/internal/application/compiled-artifacts.test.ts
@@ -39,6 +39,8 @@ describe("createWorkflowWorldPluginSource", () => {
expect(source).toContain("/compiled/@workflow/world-local/index.js");
expect(source).toContain("resolveLocalWorkflowWorldDataDirectory(process.cwd())");
expect(source).toContain("applyLocalWorkflowWorldDeliveryTimeoutDefaults");
+ expect(source).toContain("authenticateLocalWorkflowDeliveries(");
+ expect(source).toContain("recoverActiveRuns: false");
expect(source.indexOf("applyLocalWorkflowWorldDeliveryTimeoutDefaults();")).toBeLessThan(
source.indexOf("workflowWorldModule.createWorld"),
);
@@ -59,6 +61,7 @@ describe("createWorkflowWorldPluginSource", () => {
expect(source).toContain("/compiled/@workflow/world-vercel/index.js");
expect(source).toMatch(/headers: \{ "User-Agent": "eve\/.+" \}/);
expect(source).toContain("applyVercelWorkflowWorldDefaults(workflowWorld);");
+ expect(source).not.toContain("authenticateLocalWorkflowDeliveries");
expect(source.indexOf('installWorkflowDeploymentId("')).toBeGreaterThan(-1);
expect(source.indexOf('installWorkflowDeploymentId("')).toBeLessThan(
source.indexOf("await workflowWorldModule.createWorld("),
diff --git a/packages/eve/src/internal/application/compiled-artifacts.ts b/packages/eve/src/internal/application/compiled-artifacts.ts
index cf29d939e..134f584a3 100644
--- a/packages/eve/src/internal/application/compiled-artifacts.ts
+++ b/packages/eve/src/internal/application/compiled-artifacts.ts
@@ -328,17 +328,24 @@ function resolveWorkflowWorldWiring(packageName: string): WorkflowWorldWiring {
const deliveryTimeoutsImportSpecifier = stringifyEsmImportSpecifier(
resolvePackageSourceFilePath("src/internal/workflow/local-world-delivery-timeouts.ts"),
);
+ const deliveryAuthImportSpecifier = stringifyEsmImportSpecifier(
+ resolvePackageSourceFilePath("src/internal/workflow/local-world-delivery-auth.ts"),
+ );
const moduleImportSpecifier = resolvePackageCompiledFilePath(
`src/compiled/${packageName}/index.js`,
);
const importLines = `
import { applyLocalWorkflowWorldDeliveryTimeoutDefaults } from ${deliveryTimeoutsImportSpecifier};
+import { authenticateLocalWorkflowDeliveries } from ${deliveryAuthImportSpecifier};
import { resolveLocalWorkflowWorldDataDirectory } from ${dataDirectoryImportSpecifier};`.trimStart();
const createWorldSource = `
applyLocalWorkflowWorldDeliveryTimeoutDefaults();
-const workflowWorld = await workflowWorldModule.createWorld({
- dataDir: resolveLocalWorkflowWorldDataDirectory(process.cwd()),
-});`.trimStart();
+const workflowWorld = authenticateLocalWorkflowDeliveries(
+ await workflowWorldModule.createWorld({
+ dataDir: resolveLocalWorkflowWorldDataDirectory(process.cwd()),
+ recoverActiveRuns: false,
+ }),
+);`.trimStart();
return {
moduleImportSpecifier,
extraImportLines: importLines.split("\n"),
diff --git a/packages/eve/src/internal/workflow/local-world-delivery-auth.test.ts b/packages/eve/src/internal/workflow/local-world-delivery-auth.test.ts
new file mode 100644
index 000000000..d44c7f1b7
--- /dev/null
+++ b/packages/eve/src/internal/workflow/local-world-delivery-auth.test.ts
@@ -0,0 +1,136 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import {
+ authenticateLocalWorkflowDeliveries,
+ LOCAL_WORKFLOW_DELIVERY_HEADER,
+} from "#internal/workflow/local-world-delivery-auth.js";
+
+const SECRET = "test-delivery-secret-0123456789";
+const FLOW_URL = "http://localhost/.well-known/workflow/v1/flow";
+const QUEUE_HEADERS = {
+ "x-vqs-message-attempt": "1",
+ "x-vqs-message-id": "msg_1",
+ "x-vqs-queue-name": "__wkf_workflow_entry",
+};
+
+function createFakeWorld() {
+ const enqueued: Array<{
+ name: string;
+ message: unknown;
+ headers: Record<string, string> | undefined;
+ }> = [];
+ const start = vi.fn(async () => {});
+ const world = {
+ queue: async (
+ name: string,
+ message: unknown,
+ options?: { headers?: Record<string, string> },
+ ) => {
+ enqueued.push({ name, message, headers: options?.headers });
+ return { messageId: "msg_1" };
+ },
+ createQueueHandler:
+ (_prefix: string, handler: (message: unknown) => Promise<void>) =>
+ async (request: Request) => {
+ await handler(await request.json());
+ return Response.json({ ok: true });
+ },
+ runs: {
+ list: vi.fn(async ({ pagination }: { pagination: { cursor?: string } }) =>
+ pagination.cursor === undefined
+ ? {
+ cursor: "page_2",
+ data: [{ runId: "wrun_1", workflowName: "entry" }],
+ hasMore: true,
+ }
+ : {
+ cursor: null,
+ data: [{ runId: "wrun_2", workflowName: "other" }],
+ hasMore: false,
+ },
+ ),
+ },
+ start,
+ };
+ const secured = authenticateLocalWorkflowDeliveries(world as never, SECRET);
+ return { enqueued, secured, start, listRuns: world.runs.list };
+}
+
+describe("authenticateLocalWorkflowDeliveries", () => {
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ });
+
+ it("stamps every enqueue and keeps the caller's headers", async () => {
+ const { enqueued, secured } = createFakeWorld();
+ await secured.queue("__wkf_workflow_entry" as never, { runId: "wrun_1" } as never, {
+ headers: { "x-other": "1" },
+ });
+ expect(enqueued).toEqual([
+ {
+ name: "__wkf_workflow_entry",
+ message: { runId: "wrun_1" },
+ headers: { "x-other": "1", [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },
+ },
+ ]);
+ });
+
+ it("rejects a forged delivery before the handler sees its message", async () => {
+ const { secured } = createFakeWorld();
+ const inner = vi.fn(async () => {});
+ const handle = secured.createQueueHandler("__wkf_workflow_" as never, inner as never);
+ const body = JSON.stringify({
+ runInput: {
+ input: [
+ {
+ serializedContext: {
+ "eve.auth": { principalId: "victim", principalType: "user" },
+ },
+ },
+ ],
+ },
+ });
+ const post = (headers: Record<string, string>) =>
+ handle(new Request(FLOW_URL, { body, headers, method: "POST" }));
+
+ expect((await post(QUEUE_HEADERS)).status).toBe(401);
+ expect(
+ (await post({ ...QUEUE_HEADERS, [LOCAL_WORKFLOW_DELIVERY_HEADER]: "guess" })).status,
+ ).toBe(401);
+ expect(inner).not.toHaveBeenCalled();
+
+ const trusted = await post({ ...QUEUE_HEADERS, [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET });
+ expect(trusted.status).toBe(200);
+ expect(inner).toHaveBeenCalledTimes(1);
+ });
+
+ it("re-enqueues active runs through the stamped queue when the world starts", async () => {
+ vi.stubEnv("WORKFLOW_QUEUE_NAMESPACE", "evetest");
+ const { enqueued, secured, start, listRuns } = createFakeWorld();
+ await secured.start?.();
+ expect(start).toHaveBeenCalledTimes(1);
+ expect(enqueued).toEqual([
+ {
+ name: "__evetest_wkf_workflow_entry",
+ message: { runId: "wrun_1" },
+ headers: { [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },
+ },
+ {
+ name: "__evetest_wkf_workflow_other",
+ message: { runId: "wrun_2" },
+ headers: { [LOCAL_WORKFLOW_DELIVERY_HEADER]: SECRET },
+ },
+ ]);
+ expect(listRuns.mock.calls).toEqual([
+ [{ pagination: { cursor: undefined }, resolveData: "none", status: ["pending", "running"] }],
+ [{ pagination: { cursor: "page_2" }, resolveData: "none", status: ["pending", "running"] }],
+ ]);
+ });
+
+ it("rejects recovery without an installed namespace", async () => {
+ vi.stubEnv("WORKFLOW_QUEUE_NAMESPACE", undefined);
+ const { enqueued, secured } = createFakeWorld();
+ await expect(secured.start?.()).rejects.toThrow("WORKFLOW_QUEUE_NAMESPACE");
+ expect(enqueued).toEqual([]);
+ });
+});
diff --git a/packages/eve/src/internal/workflow/local-world-delivery-auth.ts b/packages/eve/src/internal/workflow/local-world-delivery-auth.ts
new file mode 100644
index 000000000..6aef407e8
--- /dev/null
+++ b/packages/eve/src/internal/workflow/local-world-delivery-auth.ts
@@ -0,0 +1,83 @@
+import { randomBytes } from "node:crypto";
+
+import type { ValidQueueName } from "#compiled/@workflow/world/index.js";
+import type { LocalWorld } from "#compiled/@workflow/world-local/index.js";
+import { timingSafeEqualStrings } from "#internal/nitro/dev-client-address.js";
+import {
+ workflowQueuePrefix,
+ WORKFLOW_QUEUE_NAMESPACE_ENV,
+} from "#internal/workflow/queue-namespace.js";
+
+export const LOCAL_WORKFLOW_DELIVERY_HEADER = "x-eve-workflow-delivery";
+
+/**
+ * Makes the local World's flow route accept only deliveries this process
+ * enqueued. The route is as reachable as the server, and world-local trusts
+ * whatever `x-vqs-*` headers and body arrive, so without a proof any client
+ * could forge a delivery, including its run context and principal.
+ *
+ * The proof is a random per-process secret that never leaves the process:
+ * the local World only delivers to the server that enqueued. world-local's
+ * own startup recovery enqueues past the wrapped queue, so the World must be
+ * created with `recoverActiveRuns: false`; `start()` recovers through the
+ * stamped queue instead.
+ *
+ * Mutates and returns `world`.
+ */
+export function authenticateLocalWorkflowDeliveries(
+ world: LocalWorld,
+ secret: string = randomBytes(32).toString("hex"),
+): LocalWorld {
+ const enqueue = world.queue.bind(world);
+ world.queue = async (queueName, message, options) =>
+ await enqueue(queueName, message, {
+ ...options,
+ headers: { ...options?.headers, [LOCAL_WORKFLOW_DELIVERY_HEADER]: secret },
+ });
+
+ const createQueueHandler = world.createQueueHandler.bind(world);
+ world.createQueueHandler = (queueNamePrefix, handler) => {
+ const deliver = createQueueHandler(queueNamePrefix, handler);
+ return async (request) => {
+ const proof = request.headers.get(LOCAL_WORKFLOW_DELIVERY_HEADER);
+ if (proof === null || !timingSafeEqualStrings(proof, secret)) {
+ return Response.json({ error: "Untrusted workflow delivery." }, { status: 401 });
+ }
+ return await deliver(request);
+ };
+ };
+
+ const start = world.start?.bind(world);
+ world.start = async () => {
+ await start?.();
+ await reenqueueActiveRuns(world);
+ };
+ return world;
+}
+
+/** Mirrors world-local's startup recovery, but through the stamped `world.queue`. */
+async function reenqueueActiveRuns(world: LocalWorld): Promise<void> {
+ const namespace = process.env[WORKFLOW_QUEUE_NAMESPACE_ENV];
+ if (!namespace) {
+ throw new Error(
+ `${WORKFLOW_QUEUE_NAMESPACE_ENV} must be installed before local workflow recovery.`,
+ );
+ }
+ const prefix = workflowQueuePrefix(namespace);
+ let cursor: string | undefined;
+ do {
+ const page = await world.runs.list({
+ pagination: { cursor },
+ resolveData: "none",
+ status: ["pending", "running"],
+ });
+ for (const run of page.data) {
+ try {
+ await world.queue(`${prefix}${run.workflowName}` as ValidQueueName, { runId: run.runId });
+ } catch (error) {
+ console.warn(`[eve] Failed to re-enqueue workflow run ${run.runId}:`, error);
+ }
+ }
+ cursor = page.hasMore ? (page.cursor ?? undefined) : undefined;
+ } while (cursor !== undefined);
+}
diff --git a/packages/eve/src/internal/workflow/queue-namespace.ts b/packages/eve/src/internal/workflow/queue-namespace.ts
index 23bf6b6e5..7a23bedc3 100644
--- a/packages/eve/src/internal/workflow/queue-namespace.ts
+++ b/packages/eve/src/internal/workflow/queue-namespace.ts
@@ -9,9 +9,14 @@ export function deriveEveWorkflowQueueNamespace(agentName: string): string {
return `eve${encodedAgentName}`;
}
+/** Formats the Workflow queue prefix for an installed namespace. */
+export function workflowQueuePrefix(namespace: string): string {
+ return `__${namespace}_wkf_workflow_`;
+}
+
/** Derives the queue prefix consumed by an eve agent's workflow handler. */
export function deriveEveWorkflowQueuePrefix(agentName: string): string {
- return `__${deriveEveWorkflowQueueNamespace(agentName)}_wkf_workflow_`;
+ return workflowQueuePrefix(deriveEveWorkflowQueueNamespace(agentName));
}
/** Derives the queue topic registered for an eve agent's workflow handler. */
diff --git a/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.ts b/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.ts
new file mode 100644
index 000000000..5b98f8eba
--- /dev/null
+++ b/packages/eve/test/scenarios/local-workflow-delivery-auth.scenario.test.ts
@@ -0,0 +1,59 @@
+import { afterEach, expect, it, vi } from "vitest";
+
+import { Client } from "../../src/client/client.js";
+import { buildApplication } from "../../src/internal/nitro/host.js";
+import { useScenarioApp } from "../../src/internal/testing/scenario-app.js";
+import { startPackagedEveStart } from "./eve-start-harness.js";
+
+const scenarioApp = useScenarioApp();
+
+afterEach(() => vi.unstubAllEnvs());
+
+it("accepts only own-queue workflow deliveries in the built production server", async () => {
+ vi.stubEnv("EVE_MOCK_AUTHORED_MODELS", "");
+ vi.stubEnv("NODE_ENV", "production");
+ const { appRoot } = await scenarioApp({
+ name: "local-workflow-delivery-auth",
+ installDependencies: true,
+ files: {
+ "agent/agent.ts": `import { defineAgent } from "eve";
+import { mockModel } from "eve/evals";
+export default defineAgent({
+ model: mockModel(() => "Noted, Alice."),
+ modelContextWindowTokens: 32_000,
+});`,
+ "agent/instructions.md": "Acknowledge Alice's messages briefly.\n",
+ "agent/channels/eve.ts": `import { httpBasic } from "eve/channels/auth";
+import { eveChannel } from "eve/channels/eve";
+export default eveChannel({ auth: httpBasic({ username: "alice", password: "fixture" }) });`,
+ },
+ });
+ await buildApplication(appRoot, { skipSandboxPrewarm: true });
+ const server = await startPackagedEveStart(appRoot);
+ try {
+ for (const proof of [undefined, "incorrect-delivery-proof"]) {
+ const response = await fetch(new URL("/.well-known/workflow/v1/flow", server.url), {
+ method: "POST",
+ body: "{}",
+ headers: proof === undefined ? {} : { "x-eve-workflow-delivery": proof },
+ signal: AbortSignal.timeout(10_000),
+ });
+ expect(response.status, server.stderr()).toBe(401);
+ expect(await response.json()).toEqual({ error: "Untrusted workflow delivery." });
+ }
+
+ const client = new Client({
+ host: server.url,
+ auth: { basic: { username: "alice", password: "fixture" } },
+ });
+ const { session } = await client.sessions.create();
+ const result = await (
+ await session.send("Alice asks you to acknowledge receipt.", {
+ signal: AbortSignal.timeout(30_000),
+ })
+ ).result();
+ expect(result.message, server.stderr()).toBe("Noted, Alice.");
+ } finally {
+ await server.stop();
+ }
+}, 120_000);