The Open Framework for Building Agents
cleanup(eve): stream every model step call (#4556)
Ben Pankowf8a020e1450e
diff --git a/packages/eve/src/execution/node-step.test.ts b/packages/eve/src/execution/node-step.test.tsindex 854e3b02c..4336b5d77 100644--- a/packages/eve/src/execution/node-step.test.ts+++ b/packages/eve/src/execution/node-step.test.ts@@ -70,7 +70,7 @@ function setupMockAgentForToolExecution(toolName: string, args: unknown): void {| undefined;const onStepEnd = settings.onStepEnd as ((...args: unknown[]) => Promise<unknown>) | undefined;- this.generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {+ this.stream = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {let preparedMessages = options.messages;if (prepareStep) {const prepared = await prepareStep({@@ -122,7 +122,10 @@ function setupMockAgentForToolExecution(toolName: string, args: unknown): void {};if (onStepEnd) await onStepEnd(result);- return { ...result, responseMessages: result.response.messages };+ return {+ fullStream: (async function* () {})(),+ responseMessages: Promise.resolve(result.response.messages),+ };});return this as unknown as ToolLoopAgent;diff --git a/packages/eve/src/harness/model-call/call.ts b/packages/eve/src/harness/model-call/call.tsindex c3828752c..d33aa2d10 100644--- a/packages/eve/src/harness/model-call/call.ts+++ b/packages/eve/src/harness/model-call/call.ts@@ -376,10 +376,12 @@ export class ModelCaller {const agent = new ToolLoopAgent(settings);try {- const result =- step.emit === undefined- ? await this.generate(agent, callMessages, hooks.stepResult)- : await this.stream(agent, callMessages, hooks.stepResult, unsettledActionToolNames);+ const result = await this.stream(+ agent,+ callMessages,+ hooks.stepResult,+ unsettledActionToolNames,+ );await attempt?.complete();return result;} catch (error) {@@ -472,31 +474,6 @@ export class ModelCaller {toolResults: [...toolResultsByCallId.values()],});}-- /** Runs the call without a listener: nothing streams. */- private async generate(- agent: ToolLoopAgent,- messages: ModelMessage[],- stepResultPromise: Promise<HarnessStepResult>,- ): Promise<HarnessStepResult> {- const { catalog, generation } = this.input;- const generateResult = await agent.generate({ abortSignal: generation.signal, messages });- throwIfTurnAborted(this.step.config.abortSignal);- generation.check();- const [stepResult, responseMessages] = toEntryStep(- await stepResultPromise,- generateResult.responseMessages,- catalog.resolve,- );- assertUsableResponse(stepResult, responseMessages, false);- return withAccumulatedResponseMessages({- responseMessages: appendMissingToolResultMessages({- append: answerSkippedToolCalls(stepResult, catalog),- responseMessages,- }),- stepResult,- });- }}/**diff --git a/packages/eve/src/harness/step-hooks.ts b/packages/eve/src/harness/step-hooks.tsindex 16a868617..79c7df790 100644--- a/packages/eve/src/harness/step-hooks.ts+++ b/packages/eve/src/harness/step-hooks.ts@@ -52,10 +52,8 @@ import { resolveConversationId } from "#shared/conversation-identity.js";// ---------------------------------------------------------------------------/**- * The subset of `StepResult` that the harness reads after a step completes.- *- * Used by both the streaming (`onStepEnd` callback) and non-streaming- * (`generateText` result) code paths.+ * The subset of `StepResult` that the harness reads after a step completes, captured by the+ * `onStepEnd` callback.*/export type HarnessStepResult = Pick<StepResult<ToolSet>,diff --git a/packages/eve/src/harness/tool-input-validation.integration.test.ts b/packages/eve/src/harness/tool-input-validation.integration.test.tsindex 60988e05c..14c1bc5d3 100644--- a/packages/eve/src/harness/tool-input-validation.integration.test.ts+++ b/packages/eve/src/harness/tool-input-validation.integration.test.ts@@ -24,6 +24,29 @@ const usage = {},};+type StreamPart =+ Awaited<ReturnType<MockLanguageModelV4["doStream"]>>["stream"] extends ReadableStream<infer Part>+ ? Part+ : never;++/** Streams one scripted response per model call, in order. */+function scriptedStreams(responses: readonly (readonly StreamPart[])[]) {+ const queue = [...responses];+ return async () => {+ const parts = queue.shift();+ if (parts === undefined) throw new Error("No scripted response left.");+ return { stream: convertArrayToReadableStream([...parts]) };+ };+}++function toolCall(toolCallId: string, toolName: string, input: string): StreamPart {+ return { input, toolCallId, toolName, type: "tool-call" };+}++function finish(unified: "length" | "stop" | "tool-calls"): StreamPart {+ return { finishReason: { raw: undefined, unified }, type: "finish", usage };+}+function findToolResult(messages: readonly ModelMessage[], toolCallId: string): unknown {for (const message of messages) {if (message.role !== "tool" || !Array.isArray(message.content)) continue;@@ -46,34 +69,16 @@ describe("framework tool input validation (real AI SDK)", () => {type: "object",} as const;const model = new MockLanguageModelV4({- doGenerate: [- {- content: [- {- input: JSON.stringify({ answer: 42 }),- toolCallId: invalidCallId,- toolName: REPLY_TOOL_NAME,- type: "tool-call",- },- ],- finishReason: { raw: undefined, unified: "tool-calls" },- usage,- warnings: [],- },- {- content: [- {- input: JSON.stringify({ answer: "done" }),- toolCallId: validCallId,- toolName: REPLY_TOOL_NAME,- type: "tool-call",- },- ],- finishReason: { raw: undefined, unified: "tool-calls" },- usage,- warnings: [],- },- ],+ doStream: scriptedStreams([+ [+ toolCall(invalidCallId, REPLY_TOOL_NAME, JSON.stringify({ answer: 42 })),+ finish("tool-calls"),+ ],+ [+ toolCall(validCallId, REPLY_TOOL_NAME, JSON.stringify({ answer: "done" })),+ finish("tool-calls"),+ ],+ ]),modelId: "final-output-validation-model",provider: "eve-integration-mock",});@@ -108,8 +113,8 @@ describe("framework tool input validation (real AI SDK)", () => {}const validStep = await invalidStep.next(invalidStep.session);- expect(model.doGenerateCalls).toHaveLength(2);- expect(findToolResult(model.doGenerateCalls[1]?.prompt ?? [], invalidCallId)).toBeDefined();+ expect(model.doStreamCalls).toHaveLength(2);+ expect(findToolResult(model.doStreamCalls[1]?.prompt ?? [], invalidCallId)).toBeDefined();expect(validStep.next).toBeNull();expect(validStep.settledTurn).toEqual({ output: { answer: "done" } });});@@ -123,32 +128,19 @@ describe("framework tool input validation (real AI SDK)", () => {])("answers complete tool calls cut short at the output limit $name", async ({ truncated }) => {const skippedCallId = "read-skipped";const model = new MockLanguageModelV4({- doGenerate: [- {- content: [- {- input: JSON.stringify({ path: "notes/alice.md" }),- toolCallId: skippedCallId,- toolName: "read_note",- type: "tool-call",- },- ...truncated.map((call) => ({- ...call,- toolName: "read_note",- type: "tool-call" as const,- })),- ],- finishReason: { raw: undefined, unified: "length" },- usage,- warnings: [],- },- {- content: [{ text: "I will read Alice's note again.", type: "text" }],- finishReason: { raw: undefined, unified: "stop" },- usage,- warnings: [],- },- ],+ doStream: scriptedStreams([+ [+ toolCall(skippedCallId, "read_note", JSON.stringify({ path: "notes/alice.md" })),+ ...truncated.map((call) => toolCall(call.toolCallId, "read_note", call.input)),+ finish("length"),+ ],+ [+ { id: "reply", type: "text-start" },+ { delta: "I will read Alice's note again.", id: "reply", type: "text-delta" },+ { id: "reply", type: "text-end" },+ finish("stop"),+ ],+ ]),modelId: "output-limit-model",provider: "eve-integration-mock",});@@ -190,8 +182,8 @@ describe("framework tool input validation (real AI SDK)", () => {const nextStep = await cutShortStep.next(cutShortStep.session);expect(execute).not.toHaveBeenCalled();- expect(model.doGenerateCalls).toHaveLength(2);- const retryPrompt = model.doGenerateCalls[1]?.prompt ?? [];+ expect(model.doStreamCalls).toHaveLength(2);+ const retryPrompt = model.doStreamCalls[1]?.prompt ?? [];expect(findToolResult(retryPrompt, skippedCallId)).toMatchObject({output: { type: "error-text", value: expect.stringContaining("finish reason: length") },});diff --git a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts b/packages/eve/src/harness/tool-loop-compaction-accounting.test.tsindex 965452394..f5b1e31d6 100644--- a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts+++ b/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts@@ -109,7 +109,7 @@ function setupMockAgentSequence(results: readonly Record<string, unknown>[]): vo) {const { onStepEnd, onStepStart, prepareStep } = settings;- const generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {+ const callModel = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {const result = queue.shift();if (result === undefined) {throw new Error("No mock ToolLoopAgent result available.");@@ -143,9 +143,8 @@ function setupMockAgentSequence(results: readonly Record<string, unknown>[]): voreturn { ...result, responseMessages: getMockResponseMessages(result) };});- this.generate = generate;this.stream = vi.fn(async (options: { messages: unknown[] }) => {- const result = await generate(options);+ const result = await callModel(options);return {fullStream: new ReadableStream({start(controller) {diff --git a/packages/eve/src/harness/tool-loop-content-filter.integration.test.ts b/packages/eve/src/harness/tool-loop-content-filter.integration.test.tsindex d62bb6a84..d05275738 100644--- a/packages/eve/src/harness/tool-loop-content-filter.integration.test.ts+++ b/packages/eve/src/harness/tool-loop-content-filter.integration.test.ts@@ -84,34 +84,4 @@ describe("content-filter reporting (real AI SDK)", () => {);},);-- it.each(["", "Alice's inventory list is"])(- "rejects a filtered generated response without retry: %j",- async (text) => {- let calls = 0;- const model = new MockLanguageModelV4({- doGenerate: async () => {- calls++;- return {- content: text === "" ? [] : [{ type: "text", text }],- finishReason,- usage,- providerMetadata,- warnings: [],- };- },- });- const step = createToolLoopHarness({- resolveModel: async () => model,- tools: new Map(),- });- await expect(- step(session(), { message: "Help Alice prepare Bob's inventory list." }),- ).rejects.toMatchObject({- name: "ContentFilteredModelResponseError",- generationId: "gen_filtered",- });- expect(calls).toBe(1);- },- );});diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex b3f6aae86..14d61a351 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -497,10 +497,6 @@ function getMockResponseMessages(result: Record<string, unknown>): Record<string: [];}-function createMockGenerateResult(result: Record<string, unknown>): Record<string, unknown> {- return { ...result, responseMessages: getMockResponseMessages(result) };-}-async function* createExplicitMockFullStream(parts: readonly Record<string, unknown>[],): AsyncIterable<Record<string, unknown>> {@@ -646,12 +642,6 @@ function setupMockAgent(result: Record<string, unknown>): void {) {const { onStepEnd } = settings;- this.generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {- await invokeMockStepStart(settings, options);- if (onStepEnd) await onStepEnd(result);- return createMockGenerateResult(result);- });-this.stream = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {await invokeMockStepStart(settings, options);const mockResult = createMockStreamResult(result);@@ -678,11 +668,6 @@ function setupMockAgentSequence(results: readonly Record<string, unknown>[]): vothrow new Error("ToolLoopAgent mock exhausted its scripted results.");}const { onStepEnd } = settings;- this.generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {- await invokeMockStepStart(settings, options);- if (onStepEnd) await onStepEnd(result);- return createMockGenerateResult(result);- });this.stream = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {await invokeMockStepStart(settings, options);const mockResult = createMockStreamResult(result);@@ -891,7 +876,6 @@ function setupMockAgentError(error: Error): void {this: Record<string, unknown>,_settings: MockAgentSettings,) {- this.generate = vi.fn().mockRejectedValue(error);this.stream = vi.fn().mockRejectedValue(error);return this as unknown as ToolLoopAgent;} as unknown as MockAgentConstructor);@@ -1303,9 +1287,9 @@ describe("createToolLoopHarness", () => {);const agent = vi.mocked(ToolLoopAgent).mock.results[0]?.value as {- generate: ReturnType<typeof vi.fn>;+ stream: ReturnType<typeof vi.fn>;};- expect(agent.generate.mock.calls[0]?.[0].messages).toEqual([+ expect(agent.stream.mock.calls[0]?.[0].messages).toEqual([{ content: [{ text: "Previous reply", type: "text" }], role: "assistant" },{ content: "Continue", kind: "user" as const, role: "user" },]);@@ -3333,7 +3317,7 @@ describe("createToolLoopHarness", () => {},);- it("throws a distinct content-filter error on the non-streaming path without reissue", async () => {+ it("throws a distinct content-filter error without reissue", async () => {setupMockAgent({finishReason: "content-filter",providerMetadata: { gateway: { generationId: "gen_filtered" } },@@ -4149,7 +4133,6 @@ describe("createToolLoopHarness", () => {});vi.mocked(ToolLoopAgent).mockImplementation(function (this: ToolLoopAgent) {this.stream = streamMock;- this.generate = streamMock;return this;} as MockAgentConstructor);@@ -4189,7 +4172,7 @@ describe("createToolLoopHarness", () => {settings: MockAgentSettings,) {const { onStepEnd, prepareStep } = settings;- this.generate = modelCallMock.mockImplementation(async (options: { messages: unknown[] }) => {+ this.stream = modelCallMock.mockImplementation(async (options: { messages: unknown[] }) => {if (prepareStep) {await prepareStep({context: undefined,@@ -4205,9 +4188,8 @@ describe("createToolLoopHarness", () => {if (onStepEnd) {void Promise.resolve().then(() => onStepEnd(success));}- return createMockGenerateResult(success);+ return createMockStreamResult(success);});- this.stream = vi.fn();return this;} as MockAgentConstructor);@@ -4363,7 +4345,6 @@ describe("createToolLoopHarness", () => {const streamMock = vi.fn();vi.mocked(ToolLoopAgent).mockImplementation(function (this: ToolLoopAgent) {this.stream = streamMock;- this.generate = streamMock;return this;} as MockAgentConstructor);@@ -4780,18 +4761,6 @@ describe("createToolLoopHarness", () => {// upstream rejection arrives. Mirror that ordering in the// mock so the test can assert step.started is emitted// exactly once across the original + retry attempts.- this.generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {- if (prepareStep) {- await prepareStep({- messages: options.messages,- steps: [],- stepNumber: 0,- model: {},- context: undefined,- });- }- throw input.failure;- });this.stream = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {if (prepareStep) {await prepareStep({@@ -4805,19 +4774,6 @@ describe("createToolLoopHarness", () => {throw input.failure;});} else {- this.generate = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {- if (prepareStep) {- await prepareStep({- messages: options.messages,- steps: [],- stepNumber: 0,- model: {},- context: undefined,- });- }- if (onStepEnd) await onStepEnd(input.successResult);- return input.successResult;- });this.stream = vi.fn().mockImplementation(async (options: { messages: unknown[] }) => {if (prepareStep) {await prepareStep({@@ -7679,14 +7635,14 @@ describe("createToolLoopHarness", () => {});it("consumes text approval shortcuts without appending them as user messages", async () => {- const generateCalls: unknown[] = [];+ const streamCalls: unknown[] = [];vi.mocked(ToolLoopAgent).mockImplementation(function (this: Record<string, unknown>,settings: MockAgentSettings,) {const { onStepEnd, prepareStep } = settings;- this.generate = vi.fn().mockImplementation(async (input: { messages: unknown[] }) => {+ this.stream = vi.fn().mockImplementation(async (input: { messages: unknown[] }) => {if (prepareStep) {await prepareStep({messages: input.messages,@@ -7696,7 +7652,7 @@ describe("createToolLoopHarness", () => {context: undefined,});}- generateCalls.push(input.messages);+ streamCalls.push(input.messages);const result = {finishReason: "stop",response: { messages: [{ content: "Approved.", role: "assistant" }] },@@ -7705,7 +7661,7 @@ describe("createToolLoopHarness", () => {toolResults: [],};if (onStepEnd) await onStepEnd(result);- return createMockGenerateResult(result);+ return createMockStreamResult(result);});return this as unknown as ToolLoopAgent;} as unknown as ConstructorParameters<typeof ToolLoopAgent> extends [infer S]@@ -7782,7 +7738,7 @@ describe("createToolLoopHarness", () => {await createToolLoopHarness(config)(session, { message: "approve" });// The answer runs the call; the model reads its result, never the text that approved it.- expect(generateCalls[0]).toEqual([+ expect(streamCalls[0]).toEqual([{content: expect.stringContaining("[Pending approvals]"),kind: "context.state",@@ -8582,9 +8538,9 @@ describe("createToolLoopHarness", () => {// Verify the model received the continuation user message, not the// trailing assistant.const instance = vi.mocked(ToolLoopAgent).mock.results.at(-1)?.value as {- generate: ReturnType<typeof vi.fn>;+ stream: ReturnType<typeof vi.fn>;};- const modelMessages = instance.generate.mock.calls[0]?.[0] as {+ const modelMessages = instance.stream.mock.calls[0]?.[0] as {messages: Array<{ role: string; content: unknown }>;};expect(modelMessages.messages.at(-1)).toEqual({@@ -8631,9 +8587,9 @@ describe("createToolLoopHarness", () => {| PromptAgentSettings| undefined;const instance = vi.mocked(ToolLoopAgent).mock.results[index]?.value as- | { generate: ReturnType<typeof vi.fn> }+ | { stream: ReturnType<typeof vi.fn> }| undefined;- const call = instance?.generate.mock.calls[0]?.[0] as+ const call = instance?.stream.mock.calls[0]?.[0] as| { messages: CapturedModelCall["messages"] }| undefined;if (settings === undefined || call === undefined) {@@ -10435,18 +10391,15 @@ describe("createToolLoopHarness", () => {expect(historyRef.size).toBe(imageBytes.byteLength);expect(historyRef.path).toMatch(/^\/workspace\/\.eve\/attachments\/[0-9a-f]{16}\/logo\.png$/);- // --- Invariant 3: the mocked ToolLoopAgent.generate saw hydrated bytes.+ // --- Invariant 3: the mocked ToolLoopAgent.stream saw hydrated bytes.//- // The mock constructor ran once; grab the generate spy (no `emit`- // was passed on the config, so the harness takes the non-streaming- // branch) and verify the messages it received had `data: Buffer`- // for the FilePart.+ // The mock constructor ran once; grab the stream spy and verify the+ // messages it received had `data: Buffer` for the FilePart.const mockInstance = vi.mocked(ToolLoopAgent).mock.results[0]?.value as {- generate: ReturnType<typeof vi.fn>;stream: ReturnType<typeof vi.fn>;};expect(mockInstance).toBeDefined();- const modelCall = mockInstance.generate.mock.calls[0]?.[0] as {+ const modelCall = mockInstance.stream.mock.calls[0]?.[0] as {messages: Array<{content: Array<{ type: string; data?: unknown; mediaType?: string }>;}>;@@ -10528,8 +10481,7 @@ describe("createToolLoopHarness", () => {.mock.results.slice(1).map((result) =>- (result.value as { generate: ReturnType<typeof vi.fn> }).generate.mock- .calls[0]?.[0] as {+ (result.value as { stream: ReturnType<typeof vi.fn> }).stream.mock.calls[0]?.[0] as {messages: ModelMessage[];},);@@ -10608,9 +10560,9 @@ describe("createToolLoopHarness", () => {expect(sandbox.writes).toHaveLength(1);expect(JSON.stringify(result.session.history)).not.toContain(base64);const agent = vi.mocked(ToolLoopAgent).mock.results[0]?.value as- | { generate: ReturnType<typeof vi.fn> }+ | { stream: ReturnType<typeof vi.fn> }| undefined;- const modelCall = agent?.generate.mock.calls[0]?.[0] as { messages: ModelMessage[] };+ const modelCall = agent?.stream.mock.calls[0]?.[0] as { messages: ModelMessage[] };expect(JSON.stringify(modelCall.messages)).toContain(base64);});@@ -10661,10 +10613,9 @@ describe("createToolLoopHarness", () => {// The model-facing content swapped the non-inlinable FilePart// for a TextPart naming the sandbox path.const mockInstance = vi.mocked(ToolLoopAgent).mock.results[0]?.value as {- generate: ReturnType<typeof vi.fn>;stream: ReturnType<typeof vi.fn>;};- const modelCall = mockInstance.generate.mock.calls[0]?.[0] as {+ const modelCall = mockInstance.stream.mock.calls[0]?.[0] as {messages: Array<{content: Array<{ type: string; text?: string; data?: unknown }>;}>;@@ -10692,10 +10643,9 @@ describe("createToolLoopHarness", () => {instructions: unknown;};const instance = vi.mocked(ToolLoopAgent).mock.results.at(-1)?.value as {- generate: ReturnType<typeof vi.fn>;stream: ReturnType<typeof vi.fn>;};- const call = (instance.generate.mock.calls[0]?.[0] ?? instance.stream.mock.calls[0]?.[0]) as {+ const call = instance.stream.mock.calls[0]?.[0] as {messages: Array<{ role: string; content: unknown }>;};return { instructions: settings.instructions, messages: call.messages };