Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix: record started task runs before publishing task.started (#5058)

Andrew Barba6752c95235f7
diff --git a/.changeset/idempotent-task-dispatch.md b/.changeset/idempotent-task-dispatch.md
new file mode 100644
index 000000000..eff226028
--- /dev/null
+++ b/.changeset/idempotent-task-dispatch.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Fix a workflow tool task running twice when the session was interrupted while publishing `task.started`, for example during a slow `task.started` hook. The session now records each task's run before it publishes the event, so a retry only publishes `task.started` again and never starts a second run.
diff --git a/packages/eve/src/execution/coordination-dispatch-shared.ts b/packages/eve/src/execution/coordination-dispatch-shared.ts
index 3781013d3..3787afe66 100644
--- a/packages/eve/src/execution/coordination-dispatch-shared.ts
+++ b/packages/eve/src/execution/coordination-dispatch-shared.ts
@@ -28,6 +28,7 @@ import {
storedProjection,
turnPosition,
} from "#harness/session-machine/view.js";
+import type { TaskStartedStreamEvent } from "#protocol/message.js";
import type { RuntimeActionResult, RuntimeWorkflowTaskRequest } from "#shared/action-types.js";
import type { SessionParent, SessionTraceRoot } from "#channel/types.js";
import type { ContextReader } from "#context/key.js";
@@ -45,7 +46,6 @@ import type { WorkflowAgentMetadata } from "#tools/workflow-definition.js";
/** Input shared by direct and Workflow-originated owner-side dispatch. */
export interface CoordinationDispatchInput {
readonly workflowToolRunOwner: WorkflowToolRunOwner;
- readonly sessionWritable: WritableStream<Uint8Array>;
readonly serializedContext: Record<string, unknown>;
readonly sessionState: DurableSessionState;
}
@@ -55,6 +55,8 @@ export interface CoordinationDispatchResult {
readonly results: readonly RuntimeActionResult[];
readonly serializedContext: Record<string, unknown>;
readonly sessionState: DurableSessionState;
+ /** The tasks the dispatch started or sent to, for the step that publishes `task.started`. */
+ readonly started: readonly TaskStartedStreamEvent["data"][];
}
/** Everything preflight produces before either step's dispatch loop runs. */
@@ -87,10 +89,8 @@ export interface PreparedCoordinationDispatch<PlanEntry = RuntimeWorkflowTaskReq
/**
* Runs every dispatch precondition that may throw — durable reads, context
- * deserialization, handle-store validation, and batch planning — before
- * the caller acquires the parent stream writer, so a preflight failure
- * never leaks the writer lock. Returns undefined when no actions are
- * pending.
+ * deserialization, handle-store validation, and batch planning — before any
+ * run starts. Returns undefined when no actions are pending.
*/
export async function prepareCoordinationDispatch(input: {
readonly serializedContext: Record<string, unknown>;
diff --git a/packages/eve/src/execution/coordination-dispatch-step.ts b/packages/eve/src/execution/coordination-dispatch-step.ts
index 86dabb255..96d0b171a 100644
--- a/packages/eve/src/execution/coordination-dispatch-step.ts
+++ b/packages/eve/src/execution/coordination-dispatch-step.ts
@@ -1,4 +1,4 @@
-/** Starts workflow-tool runs for pending coordination. */
+/** Starts workflow-tool runs for pending coordination, then reports the tasks it started. */
import {
prepareCoordinationDispatch,
@@ -6,9 +6,10 @@ import {
type CoordinationDispatchResult,
} from "#execution/coordination-dispatch-shared.js";
import { createDurableSessionState } from "#execution/durable-session-store.js";
-import { publishSessionEvents } from "#execution/publish-session-events.js";
+import { commitSessionStep, type SessionStepState } from "#execution/publish-session-events.js";
import {
withSessionStateDelta,
+ type SessionStateTransition,
type WithSessionStateDelta,
} from "#execution/session/state-delta.js";
import {
@@ -28,9 +29,7 @@ import {
resolveStepAgentLimits,
} from "#execution/agent-sessions/context.js";
import type { TaskStartedStreamEvent } from "#protocol/message.js";
-import { sessionView } from "#harness/session-machine/commit.js";
import { startTask } from "#harness/session-machine/transitions.js";
-import { storedProjection } from "#harness/session-machine/view.js";
import type { RuntimeActionResult } from "#shared/action-types.js";
import type { HarnessSessionBase } from "#harness/types.js";
@@ -38,6 +37,12 @@ type CoordinationDispatchStepInput = CoordinationDispatchInput & {
readonly action: "park";
};
+/**
+ * Starts the runs of the pending calls and records them on the session. A
+ * retry starts its runs again, since a Workflow start has no idempotency key,
+ * so this step publishes nothing and commits as soon as the runs are
+ * recorded. See {@link publishTaskStartedStep}.
+ */
export async function dispatchCoordinationStep(
input: CoordinationDispatchStepInput,
): Promise<WithSessionStateDelta<CoordinationDispatchResult>> {
@@ -45,6 +50,21 @@ export async function dispatchCoordinationStep(
return await withSessionStateDelta(input, dispatchCoordination);
}
+/**
+ * Publishes `task.started` for the tasks {@link dispatchCoordinationStep}
+ * started, before their receipts reach the turn. A retry only publishes again.
+ */
+export async function publishTaskStartedStep(
+ input: SessionStepState & { readonly started: readonly TaskStartedStreamEvent["data"][] },
+): Promise<SessionStateTransition> {
+ "use step";
+ return await withSessionStateDelta(input, ({ started, ...target }) =>
+ commitSessionStep(target, (view) => started.map((task) => startTask(view, task)), {
+ origin: "own",
+ }),
+ );
+}
+
async function dispatchCoordination(
input: CoordinationDispatchStepInput,
): Promise<CoordinationDispatchResult> {
@@ -57,6 +77,7 @@ async function dispatchCoordination(
results: [],
serializedContext: input.serializedContext,
sessionState: input.sessionState,
+ started: [],
};
}
@@ -89,21 +110,15 @@ async function dispatchCoordination(
if (dispatched.started !== undefined) started.push(dispatched.started);
}
- const published = await publishSessionEvents(
- {
- serializedContext: input.serializedContext,
- sessionState:
- nextSession === session
- ? prepared.sessionState
- : createDurableSessionState({ session: nextSession }),
- sessionWritable: input.sessionWritable,
- },
- started.flatMap(
- (task) =>
- startTask(sessionView(storedProjection(nextSession.state), nextSession.state), task).events,
- ),
- );
- return { results, ...published };
+ return {
+ results,
+ serializedContext: input.serializedContext,
+ sessionState:
+ nextSession === session
+ ? prepared.sessionState
+ : createDurableSessionState({ session: nextSession }),
+ started,
+ };
}
/**
diff --git a/packages/eve/src/execution/coordination-dispatch.integration.test.ts b/packages/eve/src/execution/coordination-dispatch.integration.test.ts
new file mode 100644
index 000000000..d32fc22a9
--- /dev/null
+++ b/packages/eve/src/execution/coordination-dispatch.integration.test.ts
@@ -0,0 +1,117 @@
+import type { AnyEventRequest } from "@workflow/world";
+import { describe, expect, it, vi } from "vitest";
+import { workflowEntry } from "#execution/session/entry.js";
+import { workflowToolRunWorkflowReference } from "#execution/workflow-runtime.js";
+import { createTestRuntime } from "#internal/testing/app-harness.js";
+import { buildSerializedContext } from "#internal/testing/entry-test-helpers.js";
+import { captureTurnEvents, filterEventsByType } from "#internal/testing/events.js";
+import { deployServiceWorkflow } from "#internal/testing/workflow-tool-fixtures.js";
+import { startSessionOwner } from "#internal/testing/workflow-test-helpers.js";
+import { getWorld } from "#internal/workflow/runtime.js";
+import { defineHook } from "#public/definitions/hook.js";
+import { defineWorkflowTool } from "#tools/workflow-definition.js";
+
+describe("coordination dispatch", () => {
+ it("starts a task's run once when the step that reports it started is retried", async () => {
+ let reported = 0;
+ const runtime = await createTestRuntime({
+ agent: { name: "task-dispatch-retry" },
+ modules: [
+ {
+ logicalPath: "tools/deploy_service.ts",
+ loadNamespace: async () => ({
+ default: defineWorkflowTool({
+ description: "Deploy a service.",
+ inputSchema: {
+ type: "object",
+ properties: { service: { type: "string" } },
+ required: ["service"],
+ },
+ task: deployServiceWorkflow,
+ }),
+ }),
+ },
+ {
+ logicalPath: "hooks/task-board.ts",
+ loadNamespace: async () => ({
+ default: defineHook({
+ events: {
+ "task.started"() {
+ reported++;
+ },
+ },
+ }),
+ }),
+ },
+ ],
+ });
+
+ await runtime.run(async () => {
+ const world = await getWorld();
+ const before = await listWorkflowToolRunIds();
+ let sessionId: string | undefined;
+ let interrupted = false;
+ // The step that ran the `task.started` hook loses its completion once, as
+ // when its process exits before the step commits, so the step runs again.
+ // With batching off, every event goes through `create`, so none can skip the fault.
+ const createBatch = world.events.createBatch;
+ world.events.createBatch = undefined;
+ const createEvent = world.events.create.bind(world.events);
+ const created = vi.spyOn(world.events, "create").mockImplementation(async (...args) => {
+ const [runId, event] = args as [string, AnyEventRequest];
+ if (
+ !interrupted &&
+ reported > 0 &&
+ runId === sessionId &&
+ event.eventType === "step_completed"
+ ) {
+ interrupted = true;
+ throw new Error("The process exited before the step committed.");
+ }
+ return await createEvent(...args);
+ });
+
+ const run = await startSessionOwner(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run deploy_service with service "api"' },
+ serializedContext: buildSerializedContext({ channelKind: "http" }),
+ },
+ ]);
+ sessionId = run.runId;
+ const stream = captureTurnEvents(run);
+ try {
+ const events = await stream.nextTurn();
+ expect(interrupted).toBe(true);
+ expect(reported).toBeGreaterThan(1);
+ expect(
+ filterEventsByType(events, "task.settled").map((event) => event.data.status),
+ ).toEqual(["completed"]);
+ const started = [...(await listWorkflowToolRunIds())].filter((id) => !before.has(id));
+ expect(started).toHaveLength(1);
+ } finally {
+ created.mockRestore();
+ world.events.createBatch = createBatch;
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ }, 60_000);
+});
+
+async function listWorkflowToolRunIds(): Promise<Set<string>> {
+ const world = await getWorld();
+ const ids = new Set<string>();
+ let cursor: string | undefined;
+ do {
+ const page = await world.runs.list({
+ pagination: { cursor, limit: 100 },
+ resolveData: "none",
+ workflowName: workflowToolRunWorkflowReference.workflowId,
+ });
+ for (const run of page.data) ids.add(run.runId);
+ cursor = page.hasMore ? (page.cursor ?? undefined) : undefined;
+ } while (cursor !== undefined);
+ return ids;
+}
diff --git a/packages/eve/src/execution/session/turn-step.test.ts b/packages/eve/src/execution/session/turn-step.test.ts
index 22c749106..74b3b3885 100644
--- a/packages/eve/src/execution/session/turn-step.test.ts
+++ b/packages/eve/src/execution/session/turn-step.test.ts
@@ -985,7 +985,6 @@ describe("dispatchCoordinationStep", () => {
await dispatchCoordinationStep({
action: "park",
workflowToolRunOwner: { inbox: "generated-owner-token" },
- sessionWritable: createTestWritable(),
serializedContext: serializeContext(ctx),
sessionState: createStubSessionState(),
});
@@ -1044,7 +1043,6 @@ describe("dispatchCoordinationStep", () => {
const dispatched = await dispatchCoordinationStep({
action: "park",
workflowToolRunOwner: { inbox: "owner-inbox" },
- sessionWritable: createTestWritable(),
serializedContext: createSerializedContext(),
sessionState,
});
diff --git a/packages/eve/src/execution/session/turn.test.ts b/packages/eve/src/execution/session/turn.test.ts
index 6f9f27660..1656697f6 100644
--- a/packages/eve/src/execution/session/turn.test.ts
+++ b/packages/eve/src/execution/session/turn.test.ts
@@ -42,7 +42,10 @@ vi.mock("#compiled/@workflow/core/index.js", async (importOriginal) => ({
...(await importOriginal()),
getWorkflowMetadata: () => ({ url: "https://parent.example" }),
}));
-vi.mock("#execution/coordination-dispatch-step.js", () => ({ dispatchCoordinationStep: vi.fn() }));
+vi.mock("#execution/coordination-dispatch-step.js", () => ({
+ dispatchCoordinationStep: vi.fn(),
+ publishTaskStartedStep: vi.fn(),
+}));
vi.mock("#execution/session/turn-step.js", () => ({
turnStep: vi.fn(),
@@ -99,10 +102,17 @@ type TurnStepWorkResult = DurableStepResult extends infer R
: never
: never;
const turnStepWork = stepWork<TurnStepInput, TurnStepWorkResult>;
-const dispatchWork = stepWork<
- Parameters<typeof dispatchCoordinationStep>[0],
- CoordinationDispatchResult
->;
+/** These dispatches start no task, so their work omits `started`. */
+function dispatchWork(
+ work: (
+ input: Parameters<typeof dispatchCoordinationStep>[0],
+ ) => Promise<Omit<CoordinationDispatchResult, "started">>,
+) {
+ return stepWork(async (input: Parameters<typeof dispatchCoordinationStep>[0]) => ({
+ ...(await work(input)),
+ started: [],
+ }));
+}
const routeWork = stepWork<Parameters<typeof routeDeliverToChildren>[0], RoutedDeliverResult>;
describe("SessionExecution checkpoints", () => {
diff --git a/packages/eve/src/execution/session/turn.ts b/packages/eve/src/execution/session/turn.ts
index cbfd1a159..50ce70f3a 100644
--- a/packages/eve/src/execution/session/turn.ts
+++ b/packages/eve/src/execution/session/turn.ts
@@ -3,7 +3,10 @@ import { sleep } from "#compiled/@workflow/core/index.js";
import type { DeliverHookPayload, SessionCapabilities, TurnCaller } from "#channel/types.js";
import type { DurableSessionState } from "#execution/durable-session-store.js";
import { cancelDescendantTurnsStep } from "#execution/cancel-descendant-turns-step.js";
-import { dispatchCoordinationStep } from "#execution/coordination-dispatch-step.js";
+import {
+ dispatchCoordinationStep,
+ publishTaskStartedStep,
+} from "#execution/coordination-dispatch-step.js";
import type { SessionInputQueue } from "#execution/session/input-queue.js";
import { taskToolResult, type TaskToolCall } from "#execution/tasks/calls.js";
import { TASK_WAIT_TOOL_NAME } from "#protocol/task-tools.js";
@@ -280,15 +283,23 @@ export class SessionExecution {
* session answers them itself.
*/
private async dispatchRuns(): Promise<readonly RuntimeActionResult[]> {
- const dispatched = await this.input.cursor.advance((state) =>
+ const { cursor } = this.input;
+ const dispatched = await cursor.advance(({ serializedContext, sessionState }) =>
dispatchCoordinationStep({
action: "park",
+ serializedContext,
+ sessionState,
workflowToolRunOwner: {
inbox: sessionInboxHookToken(sessionCommandHookToken(this.input.sessionId)),
},
- ...state,
}),
);
+ // Published in a step of its own, so a retry while it runs cannot start the runs again.
+ if (dispatched.started.length > 0) {
+ await cursor.advance((state) =>
+ publishTaskStartedStep({ ...state, started: dispatched.started }),
+ );
+ }
return dispatched.results;
}
diff --git a/packages/eve/src/execution/tasks/start.ts b/packages/eve/src/execution/tasks/start.ts
index ae4d832be..f42023f7c 100644
--- a/packages/eve/src/execution/tasks/start.ts
+++ b/packages/eve/src/execution/tasks/start.ts
@@ -39,7 +39,7 @@ const log = createLogger("execution.tasks");
type TaskDispatchInput = StartWorkflowTaskInput & { readonly taskId: string };
-/** The call's answer, and the task start the dispatch step reports for it. */
+/** The call's answer, and the task start the dispatch reports for it. */
interface TaskDispatchResult {
readonly result: RuntimeToolResultActionResult;
readonly session: HarnessSessionBase;
@@ -51,9 +51,9 @@ type TaskRunEntry = Extract<WorkflowToolRunEntry, { readonly taskId: string }>;
/**
* Starts the run for a task its model step committed, once, and answers the
- * call with its receipt and the `task.started` event the dispatch step
- * publishes. Runs inside that step, where the task's cap is enforced: past
- * it, the record is dropped and the call fails.
+ * call with its receipt and the `task.started` data the next step publishes.
+ * Runs inside the dispatch step, where the task's cap is enforced: past it,
+ * the record is dropped and the call fails.
*/
export async function startTaskRun(
input: StartWorkflowTaskInput & { readonly entry: TaskRunEntry },
diff --git a/packages/eve/src/internal/testing/entry-test-helpers.ts b/packages/eve/src/internal/testing/entry-test-helpers.ts
index dd20a1985..a2eef6681 100644
--- a/packages/eve/src/internal/testing/entry-test-helpers.ts
+++ b/packages/eve/src/internal/testing/entry-test-helpers.ts
@@ -76,6 +76,7 @@ const CONDITIONAL_STEP_NAMES = new Set([
"cancelDescendantTurnsStep",
"dispatchCoordinationStep",
"notifyCancelledTaskCallerStep",
+ "publishTaskStartedStep",
"terminateChildSessionsStep",
]);