The Open Framework for Building Agents
fix(eve): parse a workflow tool's result as JSON when its run completes (#3838)
roboeve[bot]c1880f453d52
diff --git a/.changeset/workflow-tool-output-json.md b/.changeset/workflow-tool-output-json.mdnew file mode 100644index 000000000..6838f33f1--- /dev/null+++ b/.changeset/workflow-tool-output-json.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Parse a workflow tool's result as JSON when its run completes, the way a local tool's result already is. A tool that returns `ctx.ask()`'s answer directly, after the person picked an option, no longer fails the turn with `Cannot convert undefined or null to object`: undefined properties are omitted, a tool that returns nothing reports `null`, and the same applies to `ctx.reply()` in `serve` tools. A result JSON can't carry, such as a `Date` or `Map`, now fails the call with a serialization error instead of reaching the model as `{}`.diff --git a/packages/eve/src/execution/tools/workflow/body.test.ts b/packages/eve/src/execution/tools/workflow/body.test.tsindex 94491636a..dd2a5e5ad 100644--- a/packages/eve/src/execution/tools/workflow/body.test.ts+++ b/packages/eve/src/execution/tools/workflow/body.test.ts@@ -20,24 +20,25 @@ const agentContext = {capabilities: { requestInput: true },} as Partial<AgentSessionContext> as AgentSessionContext;+const input = {+ agentContext,+ callId: "call",+ entry: { entryPoint: "execute" },+ hookToken: "control",+ input: {},+ session: {+ auth: { current: null, initiator: null },+ id: "session",+ turn: { id: "turn", sequence: 1 },+ },+ stepIndex: 0,+ toolName: "deploy",+ workflowId: "workflow//test//execute",+ owner: { send: vi.fn(), sent: 0 },+ runId: "run",+} as WorkflowBodyInput;+it("binds workflow-only methods to the run context", async () => {- const input = {- agentContext,- callId: "call",- entry: { entryPoint: "execute" },- hookToken: "control",- input: {},- session: {- auth: { current: null, initiator: null },- id: "session",- turn: { id: "turn", sequence: 1 },- },- stepIndex: 0,- toolName: "deploy",- workflowId: "workflow//test//execute",- owner: { send: vi.fn(), sent: 0 },- runId: "run",- } as WorkflowBodyInput;const question = { prompt: "Continue?" };const target = "reviewer";const session = { send: vi.fn() };@@ -60,3 +61,20 @@ it("binds workflow-only methods to the run context", async () => {output: { answer: { optionId: "yes" } },});});++// #3836: a selected option's answer carries `text: undefined`, and a tool that+// returns it as-is, or returns nothing, must still complete with JSON.+it.each([+ { expected: { optionId: "yes", status: "answered" }, returnsAnswer: true },+ { expected: null, returnsAnswer: false },+])("completes with JSON output (returns the answer: $returnsAnswer)", async (row) => {+ mocks.ask.mockResolvedValue({ optionId: "yes", status: "answered", text: undefined });+ mocks.execute.mockImplementation(async (_input, ctx: WorkflowToolContext) => {+ const answer = await ctx.ask({ prompt: "Continue?" });+ return row.returnsAnswer ? answer : undefined;+ });+ const outcome = await startCallBody(input).outcome;+ expect(outcome).toEqual({ output: row.expected, status: "completed" });+ const output = outcome.status === "completed" ? outcome.output : null;+ expect(output !== null && Object.hasOwn(output as object, "text")).toBe(false);+});diff --git a/packages/eve/src/execution/tools/workflow/body.ts b/packages/eve/src/execution/tools/workflow/body.tsindex e66f6e5d3..1fd87d7eb 100644--- a/packages/eve/src/execution/tools/workflow/body.ts+++ b/packages/eve/src/execution/tools/workflow/body.ts@@ -26,8 +26,9 @@ import type {import type { WorkflowToolRunInbox } from "#execution/tools/workflow/owner.js";import { normalizeSerializableError } from "#execution/workflow-errors.js";import { readRegisteredWorkflow } from "#execution/workflow-registry.js";+import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";import type { WorkflowToolRunEntry } from "#shared/action-types.js";-import type { JsonObject, JsonValue } from "#shared/json.js";+import { parseJsonValue, type JsonObject, type JsonValue } from "#shared/json.js";import type { ToolContext } from "#tools/definition.js";export interface WorkflowBodyDefinition {@@ -188,7 +189,7 @@ async function executeCallBody(}const entryPoint = resolveWorkflowEntryPoint<WorkflowCallEntryPoint>(input);const result = entryPoint(input.executeInput ?? input.input, ctx);- let output: JsonValue;+ let output: unknown;if (!isAsyncIterable(result)) {output = await result;} else {@@ -200,15 +201,30 @@ async function executeCallBody(await input.owner.send({ from, kind: "report", update: next.value });next = await iterator.next();}- output = (next.value as JsonValue | undefined) ?? last ?? null;+ output = next.value ?? last;}- return { output, status: "completed" };+ return { output: toWorkflowOutput(input, output), status: "completed" };} catch (error) {if (signals.interrupted) return { output: INTERRUPTED_OUTPUT, status: "completed" };return toFailedOutcome(error, signals.runSignal);}}+/**+ * An authored result as the run's JSON outcome, parsed the way a local tool's+ * result is: undefined properties are omitted, a missing result is `null`,+ * and a value JSON can't carry throws, failing the run.+ */+export function toWorkflowOutput(+ call: { readonly callId: string; readonly toolName: string },+ output: unknown,+): JsonValue {+ return withToolOutputSerializationError(+ { boundary: "execute", toolCallId: call.callId, toolName: call.toolName },+ () => parseJsonValue(output ?? null),+ );+}+/** A body's failure, or its cancellation when the run stopped. */export function toFailedOutcome(error: unknown, runSignal: AbortSignal): WorkflowToolRunOutcome {if (runSignal.aborted) {diff --git a/packages/eve/src/execution/tools/workflow/serve.test.ts b/packages/eve/src/execution/tools/workflow/serve.test.tsindex 85d8160a4..2b3a30b14 100644--- a/packages/eve/src/execution/tools/workflow/serve.test.ts+++ b/packages/eve/src/execution/tools/workflow/serve.test.ts@@ -7,6 +7,7 @@ import type { WorkflowBodyInput } from "#execution/tools/workflow/body.js";import { WORKFLOW_CANCELLATION_CLEANUP_MS } from "#execution/tools/workflow/cancellation-policy.js";import type {WorkflowBodyCommand,+ WorkflowToolRunAnswer,WorkflowToolRunMessage,WorkflowToolRunRef,} from "#execution/tools/workflow/messages.js";@@ -320,6 +321,29 @@ it("holds the outcome of a body that returns right after replying until its withawait expect(started.outcome).resolves.toEqual({ output: null, status: "completed" });});+it("replies and settles with a selected ctx.ask() answer as JSON, without its undefined text (#3836)", async () => {+ mocks.deliver.mockReset();+ const answer: WorkflowToolRunAnswer = { optionId: "yes", status: "answered", text: undefined };+ mocks.serve.mockImplementation(+ async (+ receive: WorkflowServeReceive<JsonValue>,+ ctx: WorkflowServeContext<WorkflowToolRunAnswer>,+ ) => {+ await receive();+ ctx.reply(answer);+ return answer;+ },+ );+ const started = startServeBody(input);++ const outcome = await started.outcome;+ expect(outcome).toEqual({ output: { optionId: "yes", status: "answered" }, status: "completed" });+ const output = outcome.status === "completed" ? outcome.output : null;+ expect(Object.hasOwn(output as object, "text")).toBe(false);+ const [reply] = sent("reply");+ expect(Object.hasOwn(reply?.output as object, "text")).toBe(false);+});+it.each([{body: "ignores the cancel",diff --git a/packages/eve/src/execution/tools/workflow/serve.ts b/packages/eve/src/execution/tools/workflow/serve.tsindex 08e97d7af..e55e1b451 100644--- a/packages/eve/src/execution/tools/workflow/serve.ts+++ b/packages/eve/src/execution/tools/workflow/serve.ts@@ -18,6 +18,7 @@ import {createWorkflowBodyRef,resolveWorkflowEntryPoint,toFailedOutcome,+ toWorkflowOutput,WorkflowToolRunCancelledError,type StartedWorkflowBody,type WorkflowBodyControl,@@ -385,7 +386,7 @@ async function executeServeBody(target === undefined? await resolveWorkflowEntryPoint<ServeEntryPoint>(input)(receive, ctx): await serveStub(target.scope, target.tool, receive, ctx);- outcome = { output, status: "completed" };+ outcome = { output: toWorkflowOutput(input, output), status: "completed" };} catch (error) {outcome = toFailedOutcome(error, calls.runSignal);}@@ -423,7 +424,8 @@ function createServeContext(get session() {return calls.current.session;},- reply: (output) => calls.reply(output),+ reply: (output) =>+ calls.reply(toWorkflowOutput({ callId: ctx.callId, toolName: input.toolName }, output)),};return ctx;}diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.tsindex 820c59b51..f2d6ddd8b 100644--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts@@ -21,6 +21,7 @@ import {deployServiceWorkflow,failingDeployWorkflow,reportingDeployWorkflow,+ returnAskAnswerWorkflow,stepReferenceWorkflow,workflowContextMisuseWorkflow,} from "#internal/testing/workflow-tool-fixtures.js";@@ -392,6 +393,58 @@ describe("workflow tools", () => {});}, 60_000);+ // #3836: the selected option's answer has `text: undefined`, which used to+ // fail the turn when the session projected the result for the model.+ it("completes a workflow tool that returns the selected ctx.ask answer as-is", async () => {+ const runtime = await createWorkflowToolRuntime({+ agentName: "workflow-tool-raw-answer",+ execute: returnAskAnswerWorkflow,+ toolName: "confirm_deploy",+ });++ await runtime.run(async () => {+ const run = await startSessionOwner(workflowEntry, [+ {+ kind: "initial",+ ownerDeploymentId: "dpl_inline",+ input: { message: 'Run confirm_deploy with service "api"' },+ serializedContext: buildWorkflowToolSerializedContext({+ continuationToken: "http:workflow-tool-raw-answer",+ requestInput: true,+ }),+ },+ ]);+ const stream = captureTurnEvents(run);+ try {+ const requested = await stream.nextTurn();+ const request = (+ filterEventsByType(requested, "input.requested")[0] as InputRequestedStreamEvent+ ).data.requests[0]!;+ await resumeSessionInbox(sessionCommandHookToken(run.runId), {+ kind: "send",+ payload: { inputResponses: [{ optionId: "approve", requestId: request.requestId }] },+ });++ const answered = await stream.nextTurn();+ expect(filterEventsByType(answered, "turn.failed")).toHaveLength(0);+ const result = filterEventsByType(answered, "action.result").find(+ (event) =>+ event.data.result.kind === "tool-result" &&+ event.data.result.toolName === "confirm_deploy",+ );+ expect(result?.data.result.output).toMatchObject({+ optionId: "approve",+ status: "answered",+ });+ expect(Object.hasOwn(result?.data.result.output as object, "text")).toBe(false);+ expect(filterEventsByType(answered, "turn.completed")).toHaveLength(1);+ } finally {+ stream.dispose();+ await run.cancel();+ }+ });+ }, 60_000);+it("parks the open turn again when one of two pending ctx.ask questions is answered", async () => {const runtime = await createWorkflowToolRuntime({agentName: "workflow-tool-two-asks",diff --git a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts b/packages/eve/src/internal/testing/workflow-tool-fixtures.tsindex 442db307b..7d7952eb8 100644--- a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts+++ b/packages/eve/src/internal/testing/workflow-tool-fixtures.ts@@ -157,6 +157,17 @@ export async function answerWithResponderWorkflow(return JSON.stringify({ answer, runStartPrincipal: ctx.session.auth.current?.principalId });}+/** Returns `ctx.ask()`'s answer as-is; a selected option carries `text: undefined`. */+export async function returnAskAnswerWorkflow(_input: DeployInput, ctx: WorkflowToolContext) {+ "use workflow";++ return await ctx.ask({+ display: "confirmation",+ options: [{ id: "approve", label: "Approve" }],+ prompt: "Apply this change?",+ });+}+/** Asks two questions at once, so answering one leaves the other pending. */export async function confirmTwoStepsWorkflow(input: DeployInput,