Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): preserve provider tool event ordering (#307)

Rui10a7a84ee66e
diff --git a/e2e/fixtures/agent-tools-sandbox/agent/sandbox/sandbox.ts b/e2e/fixtures/agent-tools-sandbox/agent/sandbox/sandbox.ts
index 6ab5ea2fb..3b76c26f3 100644
--- a/e2e/fixtures/agent-tools-sandbox/agent/sandbox/sandbox.ts
+++ b/e2e/fixtures/agent-tools-sandbox/agent/sandbox/sandbox.ts
@@ -44,6 +44,11 @@ export const SANDBOX_CLI_TOKEN = "eve-greet-cli-ok-R7M";
export const SANDBOX_SESSION_MARKER_PATH = "/workspace/session-marker.txt";
export const SANDBOX_SESSION_MARKER_TOKEN = "sandbox-onsession-ok-X5T";
+const FANOUT_SERVER_PORT = 43_100;
+const FANOUT_SERVER_PATH = "/workspace/eve-fanout-server.py";
+const FANOUT_SERVER_LOG_PATH = "/workspace/eve-fanout-server.log";
+const FANOUT_DELAY_MS = 2_000;
+
const CLI_SCRIPT = [
"#!/usr/bin/env python3",
"import sys",
@@ -52,6 +57,56 @@ const CLI_SCRIPT = [
"",
].join("\n");
+const FANOUT_SERVER_SCRIPT = [
+ "import json",
+ "from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer",
+ "from time import monotonic, sleep",
+ "from urllib.parse import parse_qs, urlparse",
+ "",
+ "def now_ms():",
+ " return int(monotonic() * 1000)",
+ "",
+ "class Handler(BaseHTTPRequestHandler):",
+ " def do_GET(self):",
+ " parsed = urlparse(self.path)",
+ " if parsed.path == '/health':",
+ " self.respond(200, {'ok': True})",
+ " return",
+ " if parsed.path != '/delay':",
+ " self.respond(404, {'error': 'not found'})",
+ " return",
+ "",
+ " query = parse_qs(parsed.query)",
+ " label = query.get('label', [''])[0]",
+ " search_query = query.get('q', [''])[0]",
+ " if not label:",
+ " self.respond(400, {'error': 'label is required'})",
+ " return",
+ "",
+ " received_at_ms = now_ms()",
+ ` sleep(${FANOUT_DELAY_MS} / 1000)`,
+ " self.respond(200, {",
+ " 'label': label,",
+ " 'query': search_query,",
+ " 'receivedAtMs': received_at_ms,",
+ " 'completedAtMs': now_ms(),",
+ " })",
+ "",
+ " def log_message(self, format, *args):",
+ " return",
+ "",
+ " def respond(self, status, body):",
+ " encoded = json.dumps(body).encode('utf-8')",
+ " self.send_response(status)",
+ " self.send_header('Content-Type', 'application/json')",
+ " self.send_header('Content-Length', str(len(encoded)))",
+ " self.end_headers()",
+ " self.wfile.write(encoded)",
+ "",
+ `ThreadingHTTPServer(('127.0.0.1', ${FANOUT_SERVER_PORT}), Handler).serve_forever()`,
+ "",
+].join("\n");
+
const authorSnapshotId = process.env.EVE_TEST_AUTHOR_SNAPSHOT_ID;
const backend =
authorSnapshotId !== undefined
@@ -83,5 +138,22 @@ export default defineSandbox({
path: SANDBOX_SESSION_MARKER_PATH,
content: SANDBOX_SESSION_MARKER_TOKEN,
});
+ await sandbox.writeTextFile({ path: FANOUT_SERVER_PATH, content: FANOUT_SERVER_SCRIPT });
+ const startServer = await sandbox.run({
+ command: [
+ `if ! curl -fsS http://127.0.0.1:${FANOUT_SERVER_PORT}/health >/dev/null; then`,
+ ` nohup python3 ${FANOUT_SERVER_PATH} >${FANOUT_SERVER_LOG_PATH} 2>&1 &`,
+ "fi",
+ "for attempt in $(seq 1 50); do",
+ ` if curl -fsS http://127.0.0.1:${FANOUT_SERVER_PORT}/health >/dev/null; then exit 0; fi`,
+ " sleep 0.1",
+ "done",
+ `cat ${FANOUT_SERVER_LOG_PATH} >&2`,
+ "exit 1",
+ ].join("\n"),
+ });
+ if (startServer.exitCode !== 0) {
+ throw new Error(`Fanout server failed to start: ${startServer.stderr}`);
+ }
},
});
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts
new file mode 100644
index 000000000..dbb89fa08
--- /dev/null
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/bash-curl-fanout.eval.ts
@@ -0,0 +1,182 @@
+import type { HandleMessageStreamEvent } from "eve/client";
+import { defineEval } from "eve/evals";
+
+import { FANOUT_DELAY_SERVER_URL } from "./shared.js";
+
+const BASH_TOOL = "bash";
+const MINIMUM_CURL_CALLS = 10;
+const REQUESTS = [
+ { label: "curl-01", query: "Vercel AI Gateway documentation" },
+ { label: "curl-02", query: "Anthropic Claude API documentation" },
+ { label: "curl-03", query: "OpenAI API documentation" },
+ { label: "curl-04", query: "Node.js fetch documentation" },
+ { label: "curl-05", query: "React useEffect documentation" },
+ { label: "curl-06", query: "TypeScript handbook generics" },
+ { label: "curl-07", query: "MDN Fetch API documentation" },
+ { label: "curl-08", query: "GitHub Actions documentation" },
+ { label: "curl-09", query: "AWS Lambda documentation" },
+ { label: "curl-10", query: "Google Search Central documentation" },
+] as const;
+
+interface CurlMeasurement {
+ readonly label: string;
+ readonly query: string;
+ readonly serverCompletedAtMs: number;
+ readonly serverReceivedAtMs: number;
+}
+
+export default defineEval({
+ description: "Sandbox Bash: at least ten curls each start before the preceding curl finishes.",
+ async test(t) {
+ const turn = await t.send(
+ [
+ `Call the \`${BASH_TOOL}\` tool at least ${MINIMUM_CURL_CALLS} separate times in one tool-use step.`,
+ "Run every command below at least once. If you make extra calls, repeat a command below.",
+ "Do not combine commands, use a loop, or background a process.",
+ ...REQUESTS.map((request) => `${request.label}: \`${commandFor(request)}\``),
+ "After all commands return, reply with exactly: curl fanout complete",
+ ].join("\n"),
+ );
+ turn.expectOk();
+
+ t.log(formatCurlFanoutTrace(turn.events));
+ turn.calledTool(BASH_TOOL);
+ turn.noFailedActions();
+ turn.eventsSatisfy(
+ "at least ten Bash curls each start before the preceding curl finishes",
+ (events) =>
+ consecutiveCurlStartsOverlap({
+ events,
+ expectedRequests: REQUESTS,
+ minimumCalls: MINIMUM_CURL_CALLS,
+ }),
+ );
+ },
+});
+
+function commandFor(request: (typeof REQUESTS)[number]): string {
+ const url = new URL(FANOUT_DELAY_SERVER_URL);
+ url.searchParams.set("label", request.label);
+ url.searchParams.set("q", request.query);
+
+ return `curl -fsS --max-time 30 '${url.href}'`;
+}
+
+function consecutiveCurlStartsOverlap(input: {
+ readonly events: readonly HandleMessageStreamEvent[];
+ readonly expectedRequests: readonly { readonly label: string; readonly query: string }[];
+ readonly minimumCalls: number;
+}): boolean {
+ const measurements = curlMeasurements(input.events);
+ const expectedQueryByLabel = new Map(
+ input.expectedRequests.map((request) => [request.label, request.query]),
+ );
+
+ return (
+ measurements.length >= input.minimumCalls &&
+ expectedQueryByLabel.size === input.expectedRequests.length &&
+ input.expectedRequests.every((request) =>
+ measurements.some(
+ (measurement) => measurement.label === request.label && measurement.query === request.query,
+ ),
+ ) &&
+ measurements.every(
+ (measurement) =>
+ expectedQueryByLabel.get(measurement.label) === measurement.query &&
+ measurement.serverReceivedAtMs < measurement.serverCompletedAtMs,
+ ) &&
+ consecutiveStartsOverlap(measurements)
+ );
+}
+
+function consecutiveStartsOverlap(measurements: readonly CurlMeasurement[]): boolean {
+ // The delay server provides one monotonic clock for every curl request.
+ const orderedByStart = [...measurements].sort(
+ (left, right) =>
+ left.serverReceivedAtMs - right.serverReceivedAtMs || left.label.localeCompare(right.label),
+ );
+
+ for (let index = 1; index < orderedByStart.length; index += 1) {
+ const previous = orderedByStart[index - 1];
+ const current = orderedByStart[index];
+ if (
+ previous === undefined ||
+ current === undefined ||
+ previous.serverCompletedAtMs <= current.serverReceivedAtMs
+ ) {
+ return false;
+ }
+ }
+ return true;
+}
+
+function curlMeasurements(events: readonly HandleMessageStreamEvent[]): readonly CurlMeasurement[] {
+ return events.flatMap((event) => {
+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
+ if (event.data.result.toolName !== BASH_TOOL) return [];
+
+ return parseCurlMeasurement(event.data.result.output);
+ });
+}
+
+function parseCurlMeasurement(value: unknown): readonly CurlMeasurement[] {
+ const stdout = readStringField(value, "stdout");
+ if (stdout === undefined) return [];
+
+ for (const line of stdout.split("\n")) {
+ const parsed = parseJson(line);
+ const label = readStringField(parsed, "label");
+ const query = readStringField(parsed, "query");
+ const serverReceivedAtMs = readFiniteNumberField(parsed, "receivedAtMs");
+ const serverCompletedAtMs = readFiniteNumberField(parsed, "completedAtMs");
+
+ if (
+ label !== undefined &&
+ query !== undefined &&
+ serverReceivedAtMs !== undefined &&
+ serverCompletedAtMs !== undefined
+ ) {
+ return [
+ {
+ label,
+ query,
+ serverCompletedAtMs,
+ serverReceivedAtMs,
+ },
+ ];
+ }
+ }
+ return [];
+}
+
+function formatCurlFanoutTrace(events: readonly HandleMessageStreamEvent[]): string {
+ return JSON.stringify({
+ calls: curlMeasurements(events).map((measurement) => ({
+ ...measurement,
+ serverDurationMs: measurement.serverCompletedAtMs - measurement.serverReceivedAtMs,
+ })),
+ });
+}
+
+function parseJson(value: string): unknown {
+ try {
+ return JSON.parse(value);
+ } catch {
+ return undefined;
+ }
+}
+
+function readField(value: unknown, field: string): unknown {
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined;
+ return Reflect.get(value, field);
+}
+
+function readFiniteNumberField(value: unknown, field: string): number | undefined {
+ const candidate = readField(value, field);
+ return typeof candidate === "number" && Number.isFinite(candidate) ? candidate : undefined;
+}
+
+function readStringField(value: unknown, field: string): string | undefined {
+ const candidate = readField(value, field);
+ return typeof candidate === "string" ? candidate : undefined;
+}
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/shared.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/shared.ts
index 84e3d9444..fcb60eb56 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/shared.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/shared.ts
@@ -15,6 +15,9 @@ export const SANDBOX_CLI_TOKEN = "eve-greet-cli-ok-R7M";
export const SESSION_MARKER_PATH = "/workspace/session-marker.txt";
export const SESSION_MARKER_TOKEN = "sandbox-onsession-ok-X5T";
+/** Loopback endpoint that holds each curl request long enough to measure overlap. */
+export const FANOUT_DELAY_SERVER_URL = "http://127.0.0.1:43100/delay";
+
/** Mounted from `agent/sandbox/workspace/seed-data.txt` at session start. */
export const WORKSPACE_SEED_PATH = "/workspace/seed-data.txt";
export const WORKSPACE_SEED_TOKEN = "sandbox-workspace-seed-ok-Z9W";
diff --git a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
index 60569b14c..638b428a9 100644
--- a/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/channel-events/pre-tool-narration.eval.ts
@@ -5,7 +5,6 @@ import { defineEval, type EveEvalTargetHandle } from "eve/evals";
import { satisfies } from "eve/evals/expect";
const STREAMED_ACTION_TOOL = "streamed-action";
-const OBSERVATION_TOOL = "read-channel-action-narration";
interface ChannelSessionResponse {
readonly sessionId: string;
@@ -42,6 +41,36 @@ function preToolNarration(events: readonly HandleMessageStreamEvent[]): string |
return undefined;
}
+function narratedStreamedActionOrder(events: readonly HandleMessageStreamEvent[]): boolean {
+ const requests = events.flatMap((event, eventIndex) => {
+ if (event.type !== "actions.requested") return [];
+
+ return event.data.actions.flatMap((action) => {
+ if (action.kind !== "tool-call" || action.toolName !== STREAMED_ACTION_TOOL) return [];
+ return [{ callId: action.callId, eventIndex }];
+ });
+ });
+ const results = events.flatMap((event, eventIndex) => {
+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
+ if (event.data.result.toolName !== STREAMED_ACTION_TOOL || event.data.status !== "completed") {
+ return [];
+ }
+ return [{ callId: event.data.result.callId, eventIndex }];
+ });
+
+ const [request] = requests;
+ const [result] = results;
+ return (
+ request !== undefined &&
+ result !== undefined &&
+ requests.length === 1 &&
+ results.length === 1 &&
+ request.callId === result.callId &&
+ request.eventIndex < result.eventIndex &&
+ preToolNarration(events) !== undefined
+ );
+}
+
async function postChannel(
target: EveEvalTargetHandle,
path: string,
@@ -69,8 +98,7 @@ async function postChannel(
/**
* End-to-end channel contract: the adapter receives the pre-tool completion
- * before the matching action request, then exposes that observation to the
- * next model step.
+ * before the matching action request and result.
*/
export default defineEval({
description: "Channel event smoke: pre-tool narration is visible when an action is requested.",
@@ -79,27 +107,19 @@ export default defineEval({
const started = await postChannel(t.target, "/action-narration/start", {
message:
"Before calling `streamed-action`, write one short plain-text sentence explaining the action. " +
- `Then call it exactly once with label "${token}". After it returns, call ` +
- `\`${OBSERVATION_TOOL}\` exactly once. Finally, reply with the label verbatim.`,
+ `Then call it exactly once with label "${token}". After it returns, reply with the label verbatim.`,
token,
});
const session = await t.target.attachSession(started.sessionId);
- const narration = await t.require(
+ await t.require(
preToolNarration(session.events) ?? "",
satisfies((value: string) => value.length > 0, "pre-tool narration is non-empty"),
);
- session.event("action.result", {
- count: 1,
- data: {
- result: {
- kind: "tool-result",
- output: { narration },
- toolName: OBSERVATION_TOOL,
- },
- status: "completed",
- },
- });
+ session.eventsSatisfy(
+ "pre-tool narration precedes the matching streamed action result",
+ narratedStreamedActionOrder,
+ );
session.succeeded();
t.succeeded();
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
new file mode 100644
index 000000000..9221becbc
--- /dev/null
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search-narration-ordering.eval.ts
@@ -0,0 +1,124 @@
+import type { HandleMessageStreamEvent } from "eve/client";
+import { defineEval } from "eve/evals";
+
+const TOOL_NAME = "web_search";
+
+export default defineEval({
+ description: "Provider tools: narrated and un-narrated web searches preserve event order.",
+ async test(t) {
+ const narrated = await t.send(
+ [
+ `Before calling \`${TOOL_NAME}\`, write one short visible sentence explaining that you will search.`,
+ `Then call \`${TOOL_NAME}\` exactly once to answer: Who won the 2026 NBA Finals?`,
+ "After the result returns, reply with only the winning team name. Do not call another tool.",
+ ].join("\n"),
+ );
+ narrated.expectOk();
+ narrated.calledTool(TOOL_NAME, { count: 1 });
+ narrated.noFailedActions();
+ narrated.eventsSatisfy("narration completes before the provider request and result", (events) =>
+ narratedWebSearchOrder(events),
+ );
+
+ const withoutNarration = await t
+ .newSession()
+ .send(
+ [
+ `Call \`${TOOL_NAME}\` exactly once to answer: Who won the 2025 NBA Finals?`,
+ "Do not write any visible text before the tool call.",
+ "After the result returns, reply with only the winning team name. Do not call another tool.",
+ ].join("\n"),
+ );
+ withoutNarration.expectOk();
+ withoutNarration.calledTool(TOOL_NAME, { count: 1 });
+ withoutNarration.noFailedActions();
+ withoutNarration.eventsSatisfy(
+ "provider request and result stay ordered without narration",
+ (events) => unNarratedWebSearchOrder(events),
+ );
+ },
+});
+
+interface WebSearchEventOrder {
+ readonly requestIndex: number;
+ readonly resultIndex: number;
+}
+
+function narratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
+ const order = webSearchEventOrder(events);
+ return (
+ order !== undefined &&
+ preToolNarrationExists(events, order.requestIndex) &&
+ finalMessageFollowsResult(events, order.resultIndex)
+ );
+}
+
+function unNarratedWebSearchOrder(events: readonly HandleMessageStreamEvent[]): boolean {
+ const order = webSearchEventOrder(events);
+ return (
+ order !== undefined &&
+ !preToolNarrationExists(events, order.requestIndex) &&
+ finalMessageFollowsResult(events, order.resultIndex)
+ );
+}
+
+function webSearchEventOrder(
+ events: readonly HandleMessageStreamEvent[],
+): WebSearchEventOrder | undefined {
+ const requests = events.flatMap((event, eventIndex) => {
+ if (event.type !== "actions.requested") return [];
+
+ return event.data.actions.flatMap((action) => {
+ if (action.kind !== "tool-call" || action.toolName !== TOOL_NAME) return [];
+ return [{ callId: action.callId, eventIndex }];
+ });
+ });
+ const results = events.flatMap((event, eventIndex) => {
+ if (event.type !== "action.result" || event.data.result.kind !== "tool-result") return [];
+ if (event.data.result.toolName !== TOOL_NAME) return [];
+ return [{ callId: event.data.result.callId, eventIndex }];
+ });
+
+ const [request] = requests;
+ const [result] = results;
+ if (
+ request === undefined ||
+ result === undefined ||
+ requests.length !== 1 ||
+ results.length !== 1 ||
+ request.callId !== result.callId ||
+ request.eventIndex >= result.eventIndex
+ ) {
+ return undefined;
+ }
+ return { requestIndex: request.eventIndex, resultIndex: result.eventIndex };
+}
+
+function preToolNarrationExists(
+ events: readonly HandleMessageStreamEvent[],
+ requestIndex: number,
+): boolean {
+ return events
+ .slice(0, requestIndex)
+ .some(
+ (event) =>
+ event.type === "message.completed" &&
+ event.data.finishReason === "tool-calls" &&
+ event.data.message !== null &&
+ event.data.message.trim().length > 0,
+ );
+}
+
+function finalMessageFollowsResult(
+ events: readonly HandleMessageStreamEvent[],
+ resultIndex: number,
+): boolean {
+ return events
+ .slice(resultIndex + 1)
+ .some(
+ (event) =>
+ event.type === "message.completed" &&
+ event.data.finishReason !== "tool-calls" &&
+ event.data.message !== null,
+ );
+}
diff --git a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
index bc862b4f8..87357a2fb 100644
--- a/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
+++ b/e2e/fixtures/agent-tools/evals/static-tools/web-search.eval.ts
@@ -1,13 +1,57 @@
+import type { HandleMessageStreamEvent } from "eve/client";
import { defineEval } from "eve/evals";
+const TOOL_NAME = "web_search";
+
+function providerRequestsPrecedeResults(events: readonly HandleMessageStreamEvent[]): boolean {
+ const requestIndexByCallId = new Map<string, number>();
+ const resultIndexByCallId = new Map<string, number>();
+ let requestCount = 0;
+ let resultCount = 0;
+
+ for (const [eventIndex, event] of events.entries()) {
+ if (event.type === "actions.requested") {
+ for (const action of event.data.actions) {
+ if (action.kind !== "tool-call" || action.toolName !== TOOL_NAME) continue;
+ requestCount += 1;
+ requestIndexByCallId.set(action.callId, eventIndex);
+ }
+ }
+
+ if (
+ event.type === "action.result" &&
+ event.data.result.kind === "tool-result" &&
+ event.data.result.toolName === TOOL_NAME
+ ) {
+ resultCount += 1;
+ resultIndexByCallId.set(event.data.result.callId, eventIndex);
+ }
+ }
+
+ return (
+ requestCount > 0 &&
+ requestCount === requestIndexByCallId.size &&
+ resultCount === resultIndexByCallId.size &&
+ requestIndexByCallId.size === resultIndexByCallId.size &&
+ [...requestIndexByCallId].every(([callId, requestIndex]) => {
+ const resultIndex = resultIndexByCallId.get(callId);
+ return resultIndex !== undefined && requestIndex < resultIndex;
+ })
+ );
+}
+
export default defineEval({
description: "Provider tools smoke: gateway web search answers a current-events question.",
async test(t) {
const turn = await t.send("Who won the 2026 NBA finals");
t.succeeded();
- t.calledTool("web_search");
+ t.calledTool(TOOL_NAME);
t.noFailedActions();
+ turn.eventsSatisfy(
+ "each provider web-search request is emitted once before its result",
+ (events) => providerRequestsPrecedeResults(events),
+ );
t.messageIncludes(/New York Knicks/iu);
t.judge.autoevals
.factuality("The New York Knicks won the 2026 NBA Finals.", {
diff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.ts
index b8f4c0233..61490f076 100644
--- a/packages/eve/src/harness/emission.test.ts
+++ b/packages/eve/src/harness/emission.test.ts
@@ -237,6 +237,59 @@ describe("emitStreamContent empty delivery", () => {
});
describe("emitStreamContent action requests", () => {
+ it("emits a provider action batch before any provider result arrives", async () => {
+ const events: Parameters<HarnessEmitFn>[0][] = [];
+ const emit: HarnessEmitFn = async (event) => {
+ events.push(event);
+ };
+ let releaseResults!: () => void;
+ const resultsPending = new Promise<void>((resolve) => {
+ releaseResults = resolve;
+ });
+ const searches = Array.from({ length: 10 }, (_, index) => ({
+ input: { query: `tri-state-${index + 1}` },
+ providerExecuted: true,
+ toolCallId: `search-${index + 1}`,
+ toolName: "web_search",
+ type: "tool-call" as const,
+ }));
+
+ async function* controlledStream(): AsyncIterable<TextStreamPart<ToolSet>> {
+ for (const call of searches) {
+ yield call as TextStreamPart<ToolSet>;
+ }
+ await resultsPending;
+ for (const call of searches) {
+ yield {
+ input: call.input,
+ output: { results: [] },
+ providerExecuted: true,
+ toolCallId: call.toolCallId,
+ toolName: call.toolName,
+ type: "tool-result",
+ } as TextStreamPart<ToolSet>;
+ }
+ yield { finishReason: "stop", type: "finish-step" } as TextStreamPart<ToolSet>;
+ }
+
+ const run = emitStreamContent(emit, EMISSION_STATE, controlledStream());
+ try {
+ await vi.waitFor(() => {
+ const actionRequests = events.filter((event) => event.type === "actions.requested");
+ expect(actionRequests).toHaveLength(1);
+ expect(actionRequests[0]?.data.actions.map((action) => action.callId)).toEqual(
+ searches.map((call) => call.toolCallId),
+ );
+ });
+ expect(events.some((event) => event.type === "action.result")).toBe(false);
+ } finally {
+ releaseResults();
+ }
+
+ const streamResult = await run;
+ expect([...streamResult.emittedActionCallIds]).toEqual(searches.map((call) => call.toolCallId));
+ });
+
it("completes pre-tool text before emitting a streamed action request", async () => {
const emit = createEmitStub();
const tools = new Map<string, HarnessToolDefinition>([
diff --git a/packages/eve/src/harness/emission.ts b/packages/eve/src/harness/emission.ts
index 7ef0e1e75..8355cd282 100644
--- a/packages/eve/src/harness/emission.ts
+++ b/packages/eve/src/harness/emission.ts
@@ -49,6 +49,7 @@ import type {
import { isAuthorizationSignal, isPendingAuthorizationToolOutput } from "#harness/authorization.js";
import { contextStorage } from "#context/container.js";
import { readToolInterrupt } from "#harness/tool-interrupts.js";
+import { createProviderStreamActionBatch } from "#harness/stream-actions.js";
import type {
HarnessEmitFn,
HarnessSession,
@@ -343,9 +344,9 @@ interface StreamActionEmissionOptions {
* Consumes the AI SDK `fullStream` and emits real-time text and reasoning
* events.
*
- * Emits local and provider tool results in source order. A result without a
- * streamed call resumes a call from an earlier step. `emitStepActions` emits
- * the remaining events.
+ * Emits local tool events in source order. Provider calls that arrive in one
+ * stream batch into one request event before their first result. A result
+ * without a streamed call resumes a call from an earlier step.
*/
export async function emitStreamContent(
emitFn: HarnessEmitFn,
@@ -360,6 +361,8 @@ export async function emitStreamContent(
const toolCallIdsSeenInStream = new Set<string>();
const emittedActionCallIds = new Set<string>();
const emittedActionResultCallIds = new Set<string>();
+ const providerToolCallIdsSeen = new Set<string>();
+ const providerActionBatch = createProviderStreamActionBatch({ emitFn, state });
const handledInlineToolResultCallIds = new Set<string>();
const inlineAuthorizationResults: TypedToolResult<ToolSet>[] = [];
const inlineToolResultParts: InlineToolResultPart[] = [];
@@ -400,12 +403,25 @@ export async function emitStreamContent(
);
};
- const emitProviderToolCall = async (toolCall: {
+ const collectProviderToolCall = async (toolCall: {
readonly input?: unknown;
readonly toolCallId: string;
readonly toolName: string;
}): Promise<void> => {
- await emitActionRequest({
+ if (providerToolCallIdsSeen.has(toolCall.toolCallId)) {
+ return;
+ }
+ providerToolCallIdsSeen.add(toolCall.toolCallId);
+ if (emittedActionCallIds.has(toolCall.toolCallId)) {
+ return;
+ }
+ emittedActionCallIds.add(toolCall.toolCallId);
+
+ if (currentMessage.trim().length > 0) {
+ await flushCurrentMessage();
+ }
+
+ providerActionBatch.observe({
callId: toolCall.toolCallId,
input: resolveToolCallInputObject(toolCall.input, {
callId: toolCall.toolCallId,
@@ -465,6 +481,7 @@ export async function emitStreamContent(
switch (part.type) {
case "reasoning-delta":
+ await providerActionBatch.flush();
currentReasoning += part.text;
await emitFn(
createReasoningAppendedEvent({
@@ -477,6 +494,7 @@ export async function emitStreamContent(
);
break;
case "text-delta":
+ await providerActionBatch.flush();
// Flush accumulated reasoning before text begins.
if (currentReasoning.trim().length > 0) {
await emitFn(
@@ -504,8 +522,9 @@ export async function emitStreamContent(
const toolCall = part as TypedToolCall<ToolSet>;
toolCallIdsSeenInStream.add(toolCall.toolCallId);
if (toolCall.providerExecuted === true) {
- await emitProviderToolCall(toolCall);
+ await collectProviderToolCall(toolCall);
} else {
+ await providerActionBatch.flush();
await emitToolCall(toolCall);
}
break;
@@ -517,11 +536,12 @@ export async function emitStreamContent(
break;
}
if (inlineToolResult.providerExecuted === true) {
- await emitProviderToolCall({
+ await collectProviderToolCall({
input: "input" in inlineToolResult ? inlineToolResult.input : undefined,
toolCallId: inlineToolResult.toolCallId,
toolName: inlineToolResult.toolName,
});
+ await providerActionBatch.flush();
await emitActionResult(createRuntimeToolResultFromStepResult(inlineToolResult));
// Provider results already live in the assistant response. Do not
// add a local tool message.
@@ -541,6 +561,7 @@ export async function emitStreamContent(
// An approved tool can resume with its result but no matching call in
// this step. Emit it before the message that consumes it.
+ await providerActionBatch.flush();
await flushCurrentMessage();
if (isInlineAuthorizationToolResult(inlineToolResult)) {
// Keep authorization output for the park detector instead of
@@ -567,7 +588,8 @@ export async function emitStreamContent(
case "tool-error": {
const toolError = part as TypedToolError<ToolSet>;
if (toolError.providerExecuted === true) {
- await emitProviderToolCall(toolError);
+ await collectProviderToolCall(toolError);
+ await providerActionBatch.flush();
await emitActionResult(
createRuntimeToolResultFromValue({
callId: toolError.toolCallId,
@@ -591,6 +613,7 @@ export async function emitStreamContent(
}
case "finish-step":
finishReason = normalizeAssistantStepFinishReason(part.finishReason);
+ await providerActionBatch.flush();
break;
case "error":
// `part.error` is typed as `unknown` — AI SDK providers emit
@@ -605,6 +628,8 @@ export async function emitStreamContent(
}
}
+ await providerActionBatch.flush();
+
if (streamError !== undefined) {
throw streamError;
}
diff --git a/packages/eve/src/harness/stream-actions.ts b/packages/eve/src/harness/stream-actions.ts
new file mode 100644
index 000000000..eb1a1cc25
--- /dev/null
+++ b/packages/eve/src/harness/stream-actions.ts
@@ -0,0 +1,82 @@
+import { createActionsRequestedEvent } from "#protocol/message.js";
+import type { RuntimeToolCallActionRequest } from "#runtime/actions/types.js";
+import type { HarnessEmitFn } from "#harness/types.js";
+
+interface ActionEventCoordinates {
+ readonly sequence: number;
+ readonly stepIndex: number;
+ readonly turnId: string;
+}
+
+interface ProviderStreamActionBatch {
+ flush(): Promise<void>;
+ observe(action: RuntimeToolCallActionRequest): void;
+}
+
+/** Batches provider-managed calls that arrive in one streamed model response. */
+export function createProviderStreamActionBatch(input: {
+ readonly emitFn: HarnessEmitFn;
+ readonly state: ActionEventCoordinates;
+}): ProviderStreamActionBatch {
+ const pendingActions = new Map<string, RuntimeToolCallActionRequest>();
+ let actionFlush: Promise<void> = Promise.resolve();
+ let actionFlushError: unknown;
+ let actionFlushTimer: ReturnType<typeof setTimeout> | undefined;
+ let resolveActionFlushTimer: (() => void) | undefined;
+
+ const emitPendingActions = async (): Promise<void> => {
+ if (pendingActions.size === 0) return;
+
+ const actions = [...pendingActions.values()];
+ pendingActions.clear();
+ await input.emitFn(
+ createActionsRequestedEvent({
+ actions,
+ sequence: input.state.sequence,
+ stepIndex: input.state.stepIndex,
+ turnId: input.state.turnId,
+ }),
+ );
+ };
+
+ const scheduleFlush = (): void => {
+ if (actionFlushTimer !== undefined) return;
+
+ let resolveTimer: (() => void) | undefined;
+ const timerElapsed = new Promise<void>((resolve) => {
+ resolveTimer = resolve;
+ });
+ resolveActionFlushTimer = resolveTimer;
+ actionFlushTimer = setTimeout(() => {
+ actionFlushTimer = undefined;
+ resolveActionFlushTimer = undefined;
+ resolveTimer?.();
+ }, 0);
+ actionFlush = actionFlush
+ .then(() => timerElapsed)
+ .then(emitPendingActions)
+ .catch((error: unknown) => {
+ actionFlushError ??= error;
+ });
+ };
+
+ return {
+ observe(action) {
+ pendingActions.set(action.callId, action);
+ scheduleFlush();
+ },
+ async flush() {
+ if (actionFlushTimer !== undefined) {
+ clearTimeout(actionFlushTimer);
+ actionFlushTimer = undefined;
+ const resolveTimer = resolveActionFlushTimer;
+ resolveActionFlushTimer = undefined;
+ resolveTimer?.();
+ }
+
+ await actionFlush;
+ if (actionFlushError !== undefined) throw actionFlushError;
+ await emitPendingActions();
+ },
+ };
+}
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index e70e77524..b6ed71b24 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -3959,7 +3959,7 @@ describe("createToolLoopHarness", () => {
);
});
- it("emits provider-executed web_search through normal tool action events", async () => {
+ it("does not re-emit a provider web_search when its stream call lacks the provider marker", async () => {
const toolCallId = "srvtoolu_01HhTt9QAEancMSj7jE8CXN7";
const webSearchOutput = [
{
@@ -3993,7 +3993,6 @@ describe("createToolLoopHarness", () => {
fullStreamParts: [
{
input: { query: "eve release note current" },
- providerExecuted: true,
toolCallId,
toolName: "web_search",
type: "tool-call",
@@ -4035,7 +4034,6 @@ describe("createToolLoopHarness", () => {
toolCalls: [
{
input: { query: "eve release note current" },
- providerExecuted: true,
toolCallId,
toolName: "web_search",
type: "tool-call",