The Open Framework for Building Agents
fix: record started task runs before publishing task.started (#5058)
Andrew Barba6752c95235f7
diff --git a/.changeset/idempotent-task-dispatch.md b/.changeset/idempotent-task-dispatch.mdnew file mode 100644index 000000000..eff226028--- /dev/null+++ b/.changeset/idempotent-task-dispatch.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Fix a workflow tool task running twice when the session was interrupted while publishing `task.started`, for example during a slow `task.started` hook. The session now records each task's run before it publishes the event, so a retry only publishes `task.started` again and never starts a second run.diff --git a/packages/eve/src/execution/coordination-dispatch-shared.ts b/packages/eve/src/execution/coordination-dispatch-shared.tsindex 3781013d3..3787afe66 100644--- a/packages/eve/src/execution/coordination-dispatch-shared.ts+++ b/packages/eve/src/execution/coordination-dispatch-shared.ts@@ -28,6 +28,7 @@ import {storedProjection,turnPosition,} from "#harness/session-machine/view.js";+import type { TaskStartedStreamEvent } from "#protocol/message.js";import type { RuntimeActionResult, RuntimeWorkflowTaskRequest } from "#shared/action-types.js";import type { SessionParent, SessionTraceRoot } from "#channel/types.js";import type { ContextReader } from "#context/key.js";@@ -45,7 +46,6 @@ import type { WorkflowAgentMetadata } from "#tools/workflow-definition.js";/** Input shared by direct and Workflow-originated owner-side dispatch. */export interface CoordinationDispatchInput {readonly workflowToolRunOwner: WorkflowToolRunOwner;- readonly sessionWritable: WritableStream<Uint8Array>;readonly serializedContext: Record<string, unknown>;readonly sessionState: DurableSessionState;}@@ -55,6 +55,8 @@ export interface CoordinationDispatchResult {readonly results: readonly RuntimeActionResult[];readonly serializedContext: Record<string, unknown>;readonly sessionState: DurableSessionState;+ /** The tasks the dispatch started or sent to, for the step that publishes `task.started`. */+ readonly started: readonly TaskStartedStreamEvent["data"][];}/** Everything preflight produces before either step's dispatch loop runs. */@@ -87,10 +89,8 @@ export interface PreparedCoordinationDispatch<PlanEntry = RuntimeWorkflowTaskReq/*** Runs every dispatch precondition that may throw — durable reads, context- * deserialization, handle-store validation, and batch planning — before- * the caller acquires the parent stream writer, so a preflight failure- * never leaks the writer lock. Returns undefined when no actions are- * pending.+ * deserialization, handle-store validation, and batch planning — before any+ * run starts. Returns undefined when no actions are pending.*/export async function prepareCoordinationDispatch(input: {readonly serializedContext: Record<string, unknown>;diff --git a/packages/eve/src/execution/coordination-dispatch-step.ts b/packages/eve/src/execution/coordination-dispatch-step.tsindex 86dabb255..96d0b171a 100644--- a/packages/eve/src/execution/coordination-dispatch-step.ts+++ b/packages/eve/src/execution/coordination-dispatch-step.ts@@ -1,4 +1,4 @@-/** Starts workflow-tool runs for pending coordination. */+/** Starts workflow-tool runs for pending coordination, then reports the tasks it started. */import {prepareCoordinationDispatch,@@ -6,9 +6,10 @@ import {type CoordinationDispatchResult,} from "#execution/coordination-dispatch-shared.js";import { createDurableSessionState } from "#execution/durable-session-store.js";-import { publishSessionEvents } from "#execution/publish-session-events.js";+import { commitSessionStep, type SessionStepState } from "#execution/publish-session-events.js";import {withSessionStateDelta,+ type SessionStateTransition,type WithSessionStateDelta,} from "#execution/session/state-delta.js";import {@@ -28,9 +29,7 @@ import {resolveStepAgentLimits,} from "#execution/agent-sessions/context.js";import type { TaskStartedStreamEvent } from "#protocol/message.js";-import { sessionView } from "#harness/session-machine/commit.js";import { startTask } from "#harness/session-machine/transitions.js";-import { storedProjection } from "#harness/session-machine/view.js";import type { RuntimeActionResult } from "#shared/action-types.js";import type { HarnessSessionBase } from "#harness/types.js";@@ -38,6 +37,12 @@ type CoordinationDispatchStepInput = CoordinationDispatchInput & {readonly action: "park";};+/**+ * Starts the runs of the pending calls and records them on the session. A+ * retry starts its runs again, since a Workflow start has no idempotency key,+ * so this step publishes nothing and commits as soon as the runs are+ * recorded. See {@link publishTaskStartedStep}.+ */export async function dispatchCoordinationStep(input: CoordinationDispatchStepInput,): Promise<WithSessionStateDelta<CoordinationDispatchResult>> {@@ -45,6 +50,21 @@ export async function dispatchCoordinationStep(return await withSessionStateDelta(input, dispatchCoordination);}+/**+ * Publishes `task.started` for the tasks {@link dispatchCoordinationStep}+ * started, before their receipts reach the turn. A retry only publishes again.+ */+export async function publishTaskStartedStep(+ input: SessionStepState & { readonly started: readonly TaskStartedStreamEvent["data"][] },+): Promise<SessionStateTransition> {+ "use step";+ return await withSessionStateDelta(input, ({ started, ...target }) =>+ commitSessionStep(target, (view) => started.map((task) => startTask(view, task)), {+ origin: "own",+ }),+ );+}+async function dispatchCoordination(input: CoordinationDispatchStepInput,): Promise<CoordinationDispatchResult> {@@ -57,6 +77,7 @@ async function dispatchCoordination(results: [],serializedContext: input.serializedContext,sessionState: input.sessionState,+ started: [],};}@@ -89,21 +110,15 @@ async function dispatchCoordination(if (dispatched.started !== undefined) started.push(dispatched.started);}- const published = await publishSessionEvents(- {- serializedContext: input.serializedContext,- sessionState:- nextSession === session- ? prepared.sessionState- : createDurableSessionState({ session: nextSession }),- sessionWritable: input.sessionWritable,- },- started.flatMap(- (task) =>- startTask(sessionView(storedProjection(nextSession.state), nextSession.state), task).events,- ),- );- return { results, ...published };+ return {+ results,+ serializedContext: input.serializedContext,+ sessionState:+ nextSession === session+ ? prepared.sessionState+ : createDurableSessionState({ session: nextSession }),+ started,+ };}/**diff --git a/packages/eve/src/execution/coordination-dispatch.integration.test.ts b/packages/eve/src/execution/coordination-dispatch.integration.test.tsnew file mode 100644index 000000000..d32fc22a9--- /dev/null+++ b/packages/eve/src/execution/coordination-dispatch.integration.test.ts@@ -0,0 +1,117 @@+import type { AnyEventRequest } from "@workflow/world";+import { describe, expect, it, vi } from "vitest";+import { workflowEntry } from "#execution/session/entry.js";+import { workflowToolRunWorkflowReference } from "#execution/workflow-runtime.js";+import { createTestRuntime } from "#internal/testing/app-harness.js";+import { buildSerializedContext } from "#internal/testing/entry-test-helpers.js";+import { captureTurnEvents, filterEventsByType } from "#internal/testing/events.js";+import { deployServiceWorkflow } from "#internal/testing/workflow-tool-fixtures.js";+import { startSessionOwner } from "#internal/testing/workflow-test-helpers.js";+import { getWorld } from "#internal/workflow/runtime.js";+import { defineHook } from "#public/definitions/hook.js";+import { defineWorkflowTool } from "#tools/workflow-definition.js";++describe("coordination dispatch", () => {+ it("starts a task's run once when the step that reports it started is retried", async () => {+ let reported = 0;+ const runtime = await createTestRuntime({+ agent: { name: "task-dispatch-retry" },+ modules: [+ {+ logicalPath: "tools/deploy_service.ts",+ loadNamespace: async () => ({+ default: defineWorkflowTool({+ description: "Deploy a service.",+ inputSchema: {+ type: "object",+ properties: { service: { type: "string" } },+ required: ["service"],+ },+ task: deployServiceWorkflow,+ }),+ }),+ },+ {+ logicalPath: "hooks/task-board.ts",+ loadNamespace: async () => ({+ default: defineHook({+ events: {+ "task.started"() {+ reported++;+ },+ },+ }),+ }),+ },+ ],+ });++ await runtime.run(async () => {+ const world = await getWorld();+ const before = await listWorkflowToolRunIds();+ let sessionId: string | undefined;+ let interrupted = false;+ // The step that ran the `task.started` hook loses its completion once, as+ // when its process exits before the step commits, so the step runs again.+ // With batching off, every event goes through `create`, so none can skip the fault.+ const createBatch = world.events.createBatch;+ world.events.createBatch = undefined;+ const createEvent = world.events.create.bind(world.events);+ const created = vi.spyOn(world.events, "create").mockImplementation(async (...args) => {+ const [runId, event] = args as [string, AnyEventRequest];+ if (+ !interrupted &&+ reported > 0 &&+ runId === sessionId &&+ event.eventType === "step_completed"+ ) {+ interrupted = true;+ throw new Error("The process exited before the step committed.");+ }+ return await createEvent(...args);+ });++ const run = await startSessionOwner(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: 'Run deploy_service with service "api"' },+ serializedContext: buildSerializedContext({ channelKind: "http" }),+ },+ ]);+ sessionId = run.runId;+ const stream = captureTurnEvents(run);+ try {+ const events = await stream.nextTurn();+ expect(interrupted).toBe(true);+ expect(reported).toBeGreaterThan(1);+ expect(+ filterEventsByType(events, "task.settled").map((event) => event.data.status),+ ).toEqual(["completed"]);+ const started = [...(await listWorkflowToolRunIds())].filter((id) => !before.has(id));+ expect(started).toHaveLength(1);+ } finally {+ created.mockRestore();+ world.events.createBatch = createBatch;+ stream.dispose();+ await run.cancel();+ }+ });+ }, 60_000);+});++async function listWorkflowToolRunIds(): Promise<Set<string>> {+ const world = await getWorld();+ const ids = new Set<string>();+ let cursor: string | undefined;+ do {+ const page = await world.runs.list({+ pagination: { cursor, limit: 100 },+ resolveData: "none",+ workflowName: workflowToolRunWorkflowReference.workflowId,+ });+ for (const run of page.data) ids.add(run.runId);+ cursor = page.hasMore ? (page.cursor ?? undefined) : undefined;+ } while (cursor !== undefined);+ return ids;+}diff --git a/packages/eve/src/execution/session/turn-step.test.ts b/packages/eve/src/execution/session/turn-step.test.tsindex 22c749106..74b3b3885 100644--- a/packages/eve/src/execution/session/turn-step.test.ts+++ b/packages/eve/src/execution/session/turn-step.test.ts@@ -985,7 +985,6 @@ describe("dispatchCoordinationStep", () => {await dispatchCoordinationStep({action: "park",workflowToolRunOwner: { inbox: "generated-owner-token" },- sessionWritable: createTestWritable(),serializedContext: serializeContext(ctx),sessionState: createStubSessionState(),});@@ -1044,7 +1043,6 @@ describe("dispatchCoordinationStep", () => {const dispatched = await dispatchCoordinationStep({action: "park",workflowToolRunOwner: { inbox: "owner-inbox" },- sessionWritable: createTestWritable(),serializedContext: createSerializedContext(),sessionState,});diff --git a/packages/eve/src/execution/session/turn.test.ts b/packages/eve/src/execution/session/turn.test.tsindex 6f9f27660..1656697f6 100644--- a/packages/eve/src/execution/session/turn.test.ts+++ b/packages/eve/src/execution/session/turn.test.ts@@ -42,7 +42,10 @@ vi.mock("#compiled/@workflow/core/index.js", async (importOriginal) => ({...(await importOriginal()),getWorkflowMetadata: () => ({ url: "https://parent.example" }),}));-vi.mock("#execution/coordination-dispatch-step.js", () => ({ dispatchCoordinationStep: vi.fn() }));+vi.mock("#execution/coordination-dispatch-step.js", () => ({+ dispatchCoordinationStep: vi.fn(),+ publishTaskStartedStep: vi.fn(),+}));vi.mock("#execution/session/turn-step.js", () => ({turnStep: vi.fn(),@@ -99,10 +102,17 @@ type TurnStepWorkResult = DurableStepResult extends infer R: never: never;const turnStepWork = stepWork<TurnStepInput, TurnStepWorkResult>;-const dispatchWork = stepWork<- Parameters<typeof dispatchCoordinationStep>[0],- CoordinationDispatchResult->;+/** These dispatches start no task, so their work omits `started`. */+function dispatchWork(+ work: (+ input: Parameters<typeof dispatchCoordinationStep>[0],+ ) => Promise<Omit<CoordinationDispatchResult, "started">>,+) {+ return stepWork(async (input: Parameters<typeof dispatchCoordinationStep>[0]) => ({+ ...(await work(input)),+ started: [],+ }));+}const routeWork = stepWork<Parameters<typeof routeDeliverToChildren>[0], RoutedDeliverResult>;describe("SessionExecution checkpoints", () => {diff --git a/packages/eve/src/execution/session/turn.ts b/packages/eve/src/execution/session/turn.tsindex cbfd1a159..50ce70f3a 100644--- a/packages/eve/src/execution/session/turn.ts+++ b/packages/eve/src/execution/session/turn.ts@@ -3,7 +3,10 @@ import { sleep } from "#compiled/@workflow/core/index.js";import type { DeliverHookPayload, SessionCapabilities, TurnCaller } from "#channel/types.js";import type { DurableSessionState } from "#execution/durable-session-store.js";import { cancelDescendantTurnsStep } from "#execution/cancel-descendant-turns-step.js";-import { dispatchCoordinationStep } from "#execution/coordination-dispatch-step.js";+import {+ dispatchCoordinationStep,+ publishTaskStartedStep,+} from "#execution/coordination-dispatch-step.js";import type { SessionInputQueue } from "#execution/session/input-queue.js";import { taskToolResult, type TaskToolCall } from "#execution/tasks/calls.js";import { TASK_WAIT_TOOL_NAME } from "#protocol/task-tools.js";@@ -280,15 +283,23 @@ export class SessionExecution {* session answers them itself.*/private async dispatchRuns(): Promise<readonly RuntimeActionResult[]> {- const dispatched = await this.input.cursor.advance((state) =>+ const { cursor } = this.input;+ const dispatched = await cursor.advance(({ serializedContext, sessionState }) =>dispatchCoordinationStep({action: "park",+ serializedContext,+ sessionState,workflowToolRunOwner: {inbox: sessionInboxHookToken(sessionCommandHookToken(this.input.sessionId)),},- ...state,}),);+ // Published in a step of its own, so a retry while it runs cannot start the runs again.+ if (dispatched.started.length > 0) {+ await cursor.advance((state) =>+ publishTaskStartedStep({ ...state, started: dispatched.started }),+ );+ }return dispatched.results;}diff --git a/packages/eve/src/execution/tasks/start.ts b/packages/eve/src/execution/tasks/start.tsindex ae4d832be..f42023f7c 100644--- a/packages/eve/src/execution/tasks/start.ts+++ b/packages/eve/src/execution/tasks/start.ts@@ -39,7 +39,7 @@ const log = createLogger("execution.tasks");type TaskDispatchInput = StartWorkflowTaskInput & { readonly taskId: string };-/** The call's answer, and the task start the dispatch step reports for it. */+/** The call's answer, and the task start the dispatch reports for it. */interface TaskDispatchResult {readonly result: RuntimeToolResultActionResult;readonly session: HarnessSessionBase;@@ -51,9 +51,9 @@ type TaskRunEntry = Extract<WorkflowToolRunEntry, { readonly taskId: string }>;/*** Starts the run for a task its model step committed, once, and answers the- * call with its receipt and the `task.started` event the dispatch step- * publishes. Runs inside that step, where the task's cap is enforced: past- * it, the record is dropped and the call fails.+ * call with its receipt and the `task.started` data the next step publishes.+ * Runs inside the dispatch step, where the task's cap is enforced: past it,+ * the record is dropped and the call fails.*/export async function startTaskRun(input: StartWorkflowTaskInput & { readonly entry: TaskRunEntry },diff --git a/packages/eve/src/internal/testing/entry-test-helpers.ts b/packages/eve/src/internal/testing/entry-test-helpers.tsindex dd20a1985..a2eef6681 100644--- a/packages/eve/src/internal/testing/entry-test-helpers.ts+++ b/packages/eve/src/internal/testing/entry-test-helpers.ts@@ -76,6 +76,7 @@ const CONDITIONAL_STEP_NAMES = new Set(["cancelDescendantTurnsStep","dispatchCoordinationStep","notifyCancelledTaskCallerStep",+ "publishTaskStartedStep","terminateChildSessionsStep",]);