Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): parse a workflow tool's result as JSON when its run completes (#3838)

roboeve[bot]c1880f453d52
diff --git a/.changeset/workflow-tool-output-json.md b/.changeset/workflow-tool-output-json.md
new file mode 100644
index 000000000..6838f33f1
--- /dev/null
+++ b/.changeset/workflow-tool-output-json.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Parse a workflow tool's result as JSON when its run completes, the way a local tool's result already is. A tool that returns `ctx.ask()`'s answer directly, after the person picked an option, no longer fails the turn with `Cannot convert undefined or null to object`: undefined properties are omitted, a tool that returns nothing reports `null`, and the same applies to `ctx.reply()` in `serve` tools. A result JSON can't carry, such as a `Date` or `Map`, now fails the call with a serialization error instead of reaching the model as `{}`.
diff --git a/packages/eve/src/execution/tools/workflow/body.test.ts b/packages/eve/src/execution/tools/workflow/body.test.ts
index 94491636a..dd2a5e5ad 100644
--- a/packages/eve/src/execution/tools/workflow/body.test.ts
+++ b/packages/eve/src/execution/tools/workflow/body.test.ts
@@ -20,24 +20,25 @@ const agentContext = {
capabilities: { requestInput: true },
} as Partial<AgentSessionContext> as AgentSessionContext;
+const input = {
+ agentContext,
+ callId: "call",
+ entry: { entryPoint: "execute" },
+ hookToken: "control",
+ input: {},
+ session: {
+ auth: { current: null, initiator: null },
+ id: "session",
+ turn: { id: "turn", sequence: 1 },
+ },
+ stepIndex: 0,
+ toolName: "deploy",
+ workflowId: "workflow//test//execute",
+ owner: { send: vi.fn(), sent: 0 },
+ runId: "run",
+} as WorkflowBodyInput;
+
it("binds workflow-only methods to the run context", async () => {
- const input = {
- agentContext,
- callId: "call",
- entry: { entryPoint: "execute" },
- hookToken: "control",
- input: {},
- session: {
- auth: { current: null, initiator: null },
- id: "session",
- turn: { id: "turn", sequence: 1 },
- },
- stepIndex: 0,
- toolName: "deploy",
- workflowId: "workflow//test//execute",
- owner: { send: vi.fn(), sent: 0 },
- runId: "run",
- } as WorkflowBodyInput;
const question = { prompt: "Continue?" };
const target = "reviewer";
const session = { send: vi.fn() };
@@ -60,3 +61,20 @@ it("binds workflow-only methods to the run context", async () => {
output: { answer: { optionId: "yes" } },
});
});
+
+// #3836: a selected option's answer carries `text: undefined`, and a tool that
+// returns it as-is, or returns nothing, must still complete with JSON.
+it.each([
+ { expected: { optionId: "yes", status: "answered" }, returnsAnswer: true },
+ { expected: null, returnsAnswer: false },
+])("completes with JSON output (returns the answer: $returnsAnswer)", async (row) => {
+ mocks.ask.mockResolvedValue({ optionId: "yes", status: "answered", text: undefined });
+ mocks.execute.mockImplementation(async (_input, ctx: WorkflowToolContext) => {
+ const answer = await ctx.ask({ prompt: "Continue?" });
+ return row.returnsAnswer ? answer : undefined;
+ });
+ const outcome = await startCallBody(input).outcome;
+ expect(outcome).toEqual({ output: row.expected, status: "completed" });
+ const output = outcome.status === "completed" ? outcome.output : null;
+ expect(output !== null && Object.hasOwn(output as object, "text")).toBe(false);
+});
diff --git a/packages/eve/src/execution/tools/workflow/body.ts b/packages/eve/src/execution/tools/workflow/body.ts
index e66f6e5d3..1fd87d7eb 100644
--- a/packages/eve/src/execution/tools/workflow/body.ts
+++ b/packages/eve/src/execution/tools/workflow/body.ts
@@ -26,8 +26,9 @@ import type {
import type { WorkflowToolRunInbox } from "#execution/tools/workflow/owner.js";
import { normalizeSerializableError } from "#execution/workflow-errors.js";
import { readRegisteredWorkflow } from "#execution/workflow-registry.js";
+import { withToolOutputSerializationError } from "#harness/tool-output-serialization.js";
import type { WorkflowToolRunEntry } from "#shared/action-types.js";
-import type { JsonObject, JsonValue } from "#shared/json.js";
+import { parseJsonValue, type JsonObject, type JsonValue } from "#shared/json.js";
import type { ToolContext } from "#tools/definition.js";
export interface WorkflowBodyDefinition {
@@ -188,7 +189,7 @@ async function executeCallBody(
}
const entryPoint = resolveWorkflowEntryPoint<WorkflowCallEntryPoint>(input);
const result = entryPoint(input.executeInput ?? input.input, ctx);
- let output: JsonValue;
+ let output: unknown;
if (!isAsyncIterable(result)) {
output = await result;
} else {
@@ -200,15 +201,30 @@ async function executeCallBody(
await input.owner.send({ from, kind: "report", update: next.value });
next = await iterator.next();
}
- output = (next.value as JsonValue | undefined) ?? last ?? null;
+ output = next.value ?? last;
}
- return { output, status: "completed" };
+ return { output: toWorkflowOutput(input, output), status: "completed" };
} catch (error) {
if (signals.interrupted) return { output: INTERRUPTED_OUTPUT, status: "completed" };
return toFailedOutcome(error, signals.runSignal);
}
}
+/**
+ * An authored result as the run's JSON outcome, parsed the way a local tool's
+ * result is: undefined properties are omitted, a missing result is `null`,
+ * and a value JSON can't carry throws, failing the run.
+ */
+export function toWorkflowOutput(
+ call: { readonly callId: string; readonly toolName: string },
+ output: unknown,
+): JsonValue {
+ return withToolOutputSerializationError(
+ { boundary: "execute", toolCallId: call.callId, toolName: call.toolName },
+ () => parseJsonValue(output ?? null),
+ );
+}
+
/** A body's failure, or its cancellation when the run stopped. */
export function toFailedOutcome(error: unknown, runSignal: AbortSignal): WorkflowToolRunOutcome {
if (runSignal.aborted) {
diff --git a/packages/eve/src/execution/tools/workflow/serve.test.ts b/packages/eve/src/execution/tools/workflow/serve.test.ts
index 85d8160a4..2b3a30b14 100644
--- a/packages/eve/src/execution/tools/workflow/serve.test.ts
+++ b/packages/eve/src/execution/tools/workflow/serve.test.ts
@@ -7,6 +7,7 @@ import type { WorkflowBodyInput } from "#execution/tools/workflow/body.js";
import { WORKFLOW_CANCELLATION_CLEANUP_MS } from "#execution/tools/workflow/cancellation-policy.js";
import type {
WorkflowBodyCommand,
+ WorkflowToolRunAnswer,
WorkflowToolRunMessage,
WorkflowToolRunRef,
} from "#execution/tools/workflow/messages.js";
@@ -320,6 +321,29 @@ it("holds the outcome of a body that returns right after replying until its with
await expect(started.outcome).resolves.toEqual({ output: null, status: "completed" });
});
+it("replies and settles with a selected ctx.ask() answer as JSON, without its undefined text (#3836)", async () => {
+ mocks.deliver.mockReset();
+ const answer: WorkflowToolRunAnswer = { optionId: "yes", status: "answered", text: undefined };
+ mocks.serve.mockImplementation(
+ async (
+ receive: WorkflowServeReceive<JsonValue>,
+ ctx: WorkflowServeContext<WorkflowToolRunAnswer>,
+ ) => {
+ await receive();
+ ctx.reply(answer);
+ return answer;
+ },
+ );
+ const started = startServeBody(input);
+
+ const outcome = await started.outcome;
+ expect(outcome).toEqual({ output: { optionId: "yes", status: "answered" }, status: "completed" });
+ const output = outcome.status === "completed" ? outcome.output : null;
+ expect(Object.hasOwn(output as object, "text")).toBe(false);
+ const [reply] = sent("reply");
+ expect(Object.hasOwn(reply?.output as object, "text")).toBe(false);
+});
+
it.each([
{
body: "ignores the cancel",
diff --git a/packages/eve/src/execution/tools/workflow/serve.ts b/packages/eve/src/execution/tools/workflow/serve.ts
index 08e97d7af..e55e1b451 100644
--- a/packages/eve/src/execution/tools/workflow/serve.ts
+++ b/packages/eve/src/execution/tools/workflow/serve.ts
@@ -18,6 +18,7 @@ import {
createWorkflowBodyRef,
resolveWorkflowEntryPoint,
toFailedOutcome,
+ toWorkflowOutput,
WorkflowToolRunCancelledError,
type StartedWorkflowBody,
type WorkflowBodyControl,
@@ -385,7 +386,7 @@ async function executeServeBody(
target === undefined
? await resolveWorkflowEntryPoint<ServeEntryPoint>(input)(receive, ctx)
: await serveStub(target.scope, target.tool, receive, ctx);
- outcome = { output, status: "completed" };
+ outcome = { output: toWorkflowOutput(input, output), status: "completed" };
} catch (error) {
outcome = toFailedOutcome(error, calls.runSignal);
}
@@ -423,7 +424,8 @@ function createServeContext(
get session() {
return calls.current.session;
},
- reply: (output) => calls.reply(output),
+ reply: (output) =>
+ calls.reply(toWorkflowOutput({ callId: ctx.callId, toolName: input.toolName }, output)),
};
return ctx;
}
diff --git a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
index 820c59b51..f2d6ddd8b 100644
--- a/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
+++ b/packages/eve/src/execution/tools/workflow/workflow-tool-run.integration.test.ts
@@ -21,6 +21,7 @@ import {
deployServiceWorkflow,
failingDeployWorkflow,
reportingDeployWorkflow,
+ returnAskAnswerWorkflow,
stepReferenceWorkflow,
workflowContextMisuseWorkflow,
} from "#internal/testing/workflow-tool-fixtures.js";
@@ -392,6 +393,58 @@ describe("workflow tools", () => {
});
}, 60_000);
+ // #3836: the selected option's answer has `text: undefined`, which used to
+ // fail the turn when the session projected the result for the model.
+ it("completes a workflow tool that returns the selected ctx.ask answer as-is", async () => {
+ const runtime = await createWorkflowToolRuntime({
+ agentName: "workflow-tool-raw-answer",
+ execute: returnAskAnswerWorkflow,
+ toolName: "confirm_deploy",
+ });
+
+ await runtime.run(async () => {
+ const run = await startSessionOwner(workflowEntry, [
+ {
+ kind: "initial",
+ ownerDeploymentId: "dpl_inline",
+ input: { message: 'Run confirm_deploy with service "api"' },
+ serializedContext: buildWorkflowToolSerializedContext({
+ continuationToken: "http:workflow-tool-raw-answer",
+ requestInput: true,
+ }),
+ },
+ ]);
+ const stream = captureTurnEvents(run);
+ try {
+ const requested = await stream.nextTurn();
+ const request = (
+ filterEventsByType(requested, "input.requested")[0] as InputRequestedStreamEvent
+ ).data.requests[0]!;
+ await resumeSessionInbox(sessionCommandHookToken(run.runId), {
+ kind: "send",
+ payload: { inputResponses: [{ optionId: "approve", requestId: request.requestId }] },
+ });
+
+ const answered = await stream.nextTurn();
+ expect(filterEventsByType(answered, "turn.failed")).toHaveLength(0);
+ const result = filterEventsByType(answered, "action.result").find(
+ (event) =>
+ event.data.result.kind === "tool-result" &&
+ event.data.result.toolName === "confirm_deploy",
+ );
+ expect(result?.data.result.output).toMatchObject({
+ optionId: "approve",
+ status: "answered",
+ });
+ expect(Object.hasOwn(result?.data.result.output as object, "text")).toBe(false);
+ expect(filterEventsByType(answered, "turn.completed")).toHaveLength(1);
+ } finally {
+ stream.dispose();
+ await run.cancel();
+ }
+ });
+ }, 60_000);
+
it("parks the open turn again when one of two pending ctx.ask questions is answered", async () => {
const runtime = await createWorkflowToolRuntime({
agentName: "workflow-tool-two-asks",
diff --git a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts b/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
index 442db307b..7d7952eb8 100644
--- a/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
+++ b/packages/eve/src/internal/testing/workflow-tool-fixtures.ts
@@ -157,6 +157,17 @@ export async function answerWithResponderWorkflow(
return JSON.stringify({ answer, runStartPrincipal: ctx.session.auth.current?.principalId });
}
+/** Returns `ctx.ask()`'s answer as-is; a selected option carries `text: undefined`. */
+export async function returnAskAnswerWorkflow(_input: DeployInput, ctx: WorkflowToolContext) {
+ "use workflow";
+
+ return await ctx.ask({
+ display: "confirmation",
+ options: [{ id: "approve", label: "Approve" }],
+ prompt: "Apply this change?",
+ });
+}
+
/** Asks two questions at once, so answering one leaves the other pending. */
export async function confirmTwoStepsWorkflow(
input: DeployInput,