Multiplayer agent harness for work.
Stream a run's tool calls and results as AG-UI events (#1607)
Tom Firthd42cf2a875c2
diff --git a/plugins/web-ui/test/run-connection.test.ts b/plugins/web-ui/test/run-connection.test.tsindex d7dc252b4..2b84b956a 100644--- a/plugins/web-ui/test/run-connection.test.ts+++ b/plugins/web-ui/test/run-connection.test.ts@@ -193,6 +193,22 @@ test("TanStack assembles snapshot hydration and overlapping Unicode deltas exactassert.equal(state.polls, 0);});+test("tool call events on the run stream leave the streamed reply untouched", async (t) => {+ const state = streamingFetch(t);+ const stream = await makeRunResumeStreamFn("r")(model, {} as Context, {});+ await flush();+ state.send(+ { type: "TOOL_CALL_START", toolCallId: "c1", toolCallName: "execute", args: { command: "ls" } },+ "tool:c1:start",+ );+ state.send({ type: "CUSTOM", name: "run", value: { status: "running", result: null, partial: "hello" } });+ state.send({ type: "TOOL_CALL_RESULT", toolCallId: "c1", content: "a.txt", isError: false }, "tool:c1:result");+ state.send({ type: "CUSTOM", name: "delta", value: { offset: 5, delta: " world" } }, "text:11");+ state.send({ type: "CUSTOM", name: "run", value: { status: "done", result: { status: "ok" } } });+ assert.deepEqual((await stream.result()).content, [{ type: "text", text: "hello world" }]);+ assert.equal(state.polls, 0);+});+test("TanStack reconnects after a transport drop and deduplicates replayed offsets", async (t) => {const state = streamingFetch(t);const stream = await makeRunResumeStreamFn("r")(model, {} as Context, {});diff --git a/src/api/app-turn.ts b/src/api/app-turn.tsindex 69aba1ca6..a25209755 100644--- a/src/api/app-turn.ts+++ b/src/api/app-turn.ts@@ -60,6 +60,7 @@ export function createTurnMethods(| "listSessionApprovals"| "pendingApprovalForThread"| "getRun"+ | "getRunToolEntries"| "subscribeRun"| "syncRunStream"| "activeRunForThread"@@ -727,6 +728,24 @@ export function createTurnMethods(};},+ async getRunToolEntries(runId, viewer, afterSeq) {+ const run = await deps.runs.get(runId);+ if (!run || run.turnUserSeq === null) return [];+ if (viewer && !(await viewerMayUseRun(run, viewer))) return [];+ const session = await deps.sessions.getByThread(run.sessionId);+ if (!session) return [];+ const entries = await deps.sessions.getEntries(session.id, {+ sinceSeq: Math.max(run.turnUserSeq, (afterSeq ?? -1) + 1),+ });+ const nextRun = entries.findIndex((e) => {+ const owner = (e.payload as { runId?: unknown } | null)?.runId;+ return e.type === "user" && typeof owner === "string" && owner !== runId;+ });+ return entries+ .slice(0, nextRun < 0 ? undefined : nextRun)+ .filter((e) => e.type === "tool_call" || e.type === "tool_result");+ },+async stopConversation(threadRef, viewer) {const stop = async () => {const session = await deps.sessions.getByThread(threadRef);diff --git a/src/api/app-types.ts b/src/api/app-types.tsindex 60c368d9a..c71c37d64 100644--- a/src/api/app-types.ts+++ b/src/api/app-types.ts@@ -304,6 +304,7 @@ export interface App {startedAt: number | null;finishedAt: number | null;} | null>;+ getRunToolEntries(runId: string, viewer?: string, afterSeq?: number): Promise<SessionEntry[]>;stopConversation(threadRef: string, viewer?: string): Promise<boolean>;activeRunForThread(threadRef: string,diff --git a/src/api/routes/run-events.ts b/src/api/routes/run-events.tsindex f420ccf34..7b9242753 100644--- a/src/api/routes/run-events.ts+++ b/src/api/routes/run-events.ts@@ -1,9 +1,36 @@import { sendJson } from "../http.ts";+import type { SessionEntry } from "../../types.ts";import type { ApiCtx, Route } from "./route.ts";const HEARTBEAT_MS = 15_000;const MAX_BUFFER_BYTES = 1_048_576;+function toolEvent(entry: SessionEntry): { id: string; data: Record<string, unknown> } | null {+ const payload = (entry.payload ?? {}) as Record<string, unknown>;+ const { tool, callId, ...rest } = payload;+ if (typeof callId !== "string" || !callId) return null;+ if (entry.type === "tool_call") {+ return {+ id: `tool:${callId}:start`,+ data: {+ type: "TOOL_CALL_START",+ toolCallId: callId,+ toolCallName: typeof tool === "string" ? tool : "tool",+ args: rest,+ },+ };+ }+ return {+ id: `tool:${callId}:result`,+ data: {+ type: "TOOL_CALL_RESULT",+ toolCallId: callId,+ content: String(payload.result ?? ""),+ isError: payload.isError === true,+ },+ };+}+async function streamRun(ctx: ApiCtx): Promise<void> {const { app, req, res, actor } = ctx;const runId = ctx.params.id!;@@ -15,6 +42,8 @@ async function streamRun(ctx: ApiCtx): Promise<void> {let syncing = false;let refreshing = false;let refreshAgain = false;+ const sentToolEvents = new Set<string>();+ let toolCursor: number | undefined;const cleanup = (): void => {closed = true;clearInterval(heartbeat);@@ -28,6 +57,16 @@ async function streamRun(ctx: ApiCtx): Promise<void> {}res.write(`${id === undefined ? "" : `id: ${id}\n`}data: ${JSON.stringify(data)}\n\n`);};+ const drained = (): Promise<void> =>+ new Promise((resolve) => {+ const done = (): void => {+ res.off("drain", done);+ res.off("close", done);+ resolve();+ };+ res.on("drain", done);+ res.on("close", done);+ });const sync = (): void => {if (syncing || closed) return;syncing = true;@@ -36,16 +75,33 @@ async function streamRun(ctx: ApiCtx): Promise<void> {if (!closed) app.syncRunStream(runId, offset);});};- const snapshot = (run: NonNullable<typeof initial>): void => {+ const sendDrained = async (data: unknown, id?: string): Promise<void> => {+ if (res.writableNeedDrain) await drained();+ send(data, id);+ };+ const snapshot = async (run: NonNullable<typeof initial>): Promise<void> => {+ const toolEntries = await app.getRunToolEntries(runId, actor?.p, toolCursor);+ if (closed) return;offset = Math.max(offset, run.partial?.length ?? 0);- send({ type: "CUSTOM", name: "run", value: run });+ for (const entry of toolEntries) {+ const event = toolEvent(entry);+ if (event && !sentToolEvents.has(event.id)) {+ await sendDrained(event.data, event.id);+ if (closed) return;+ sentToolEvents.add(event.id);+ }+ toolCursor = entry.seq;+ }+ await sendDrained({ type: "CUSTOM", name: "run", value: run });+ if (closed) return;if (run.status === "done" || run.status === "failed" || run.result !== null) {- send({ type: "RUN_FINISHED", threadId: runId, runId }, "done");+ await sendDrained({ type: "RUN_FINISHED", threadId: runId, runId }, "done");+ if (closed) return;res.end();cleanup();}};- const refresh = async (): Promise<void> => {+ const refresh = async (hydrated?: typeof initial): Promise<void> => {if (closed) return;if (refreshing) {refreshAgain = true;@@ -55,13 +111,14 @@ async function streamRun(ctx: ApiCtx): Promise<void> {try {do {refreshAgain = false;- const run = await app.getRun(runId, actor?.p);+ const run = hydrated ?? (await app.getRun(runId, actor?.p));+ hydrated = undefined;if (closed) return;if (!run) {res.destroy();return;}- snapshot(run);+ await snapshot(run);} while (refreshAgain && !closed);} catch {res.destroy();@@ -107,12 +164,8 @@ async function streamRun(ctx: ApiCtx): Promise<void> {"x-accel-buffering": "no",});send({ type: "RUN_STARTED", threadId: runId, runId });- snapshot(initial);- // Subscribe before the second snapshot so hydration cannot miss a publication.- if (!closed) {- sync();- await refresh();- }+ await refresh(initial);+ if (!closed) await refresh();}export const runEventRoutes: ReadonlyArray<Route<ApiCtx>> = [diff --git a/test/run-events.test.ts b/test/run-events.test.tsindex 3ad3657f6..8e9629678 100644--- a/test/run-events.test.ts+++ b/test/run-events.test.ts@@ -7,6 +7,7 @@ import type { ApiCtx } from "../src/api/routes/route.ts";import type { App } from "../src/api/app.ts";import { createMemoryEventBus } from "../src/util/event-bus.ts";import { emitRunText, type RunStreamEvent } from "../src/runs/run-stream-events.ts";+import type { SessionEntry } from "../src/types.ts";const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));@@ -31,7 +32,10 @@ test("run events push deltas without snapshot polling and recover a separate worsyncRunStream(runId, offset) {emitRunText(bus, runId, workerText.slice(offset), offset);},- } as Pick<App, "getRun" | "subscribeRun" | "syncRunStream">;+ async getRunToolEntries() {+ return [];+ },+ } as Pick<App, "getRun" | "getRunToolEntries" | "subscribeRun" | "syncRunStream">;const server = createServer((req, res) => {void runEventRoutes[0]!.handle({ app, req, res, params: { id: "run" } } as unknown as ApiCtx);});@@ -103,7 +107,10 @@ test("run events deliver a pending activity snapshot before subsequent text andsyncRunStream(runId, offset) {emitRunText(bus, runId, workerText.slice(offset), offset);},- } as Pick<App, "getRun" | "subscribeRun" | "syncRunStream">;+ async getRunToolEntries() {+ return [];+ },+ } as Pick<App, "getRun" | "getRunToolEntries" | "subscribeRun" | "syncRunStream">;const server = createServer((req, res) => {void runEventRoutes[0]!.handle({ app, req, res, params: { id: "run" } } as unknown as ApiCtx);});@@ -146,3 +153,250 @@ test("run events deliver a pending activity snapshot before subsequent text andawait new Promise<void>((resolve) => server.close(() => resolve()));}});++function toolEntry(seq: number, type: "tool_call" | "tool_result", payload: Record<string, unknown>): SessionEntry {+ return {+ sessionId: "session",+ seq,+ parentSeq: null,+ type,+ payload,+ scopeLabel: "dm:U1",+ createdAt: seq,+ } as SessionEntry;+}++function frames(wire: string): Array<{ id?: string; data: { type: string; name?: string } & Record<string, unknown> }> {+ return wire+ .split("\n\n")+ .filter(Boolean)+ .map((frame) => {+ const lines = frame.split("\n");+ const id = lines.find((line) => line.startsWith("id: "))?.slice(4);+ const data = JSON.parse(lines.find((line) => line.startsWith("data: "))!.slice(6));+ return id === undefined ? { data } : { id, data };+ });+}++function toolFrames(wire: string) {+ return frames(wire).filter((frame) => frame.data.type.startsWith("TOOL_CALL_"));+}++async function streamToolRun(entries: SessionEntry[], state: { status: "running" | "done" }) {+ const bus = createMemoryEventBus<RunStreamEvent>("tool-test");+ const viewers: Array<string | undefined> = [];+ const reads: Array<{ afterSeq: number | undefined; rows: number }> = [];+ const app = {+ async getRun() {+ return {+ status: state.status,+ startedAt: 0,+ finishedAt: null,+ result: state.status === "done" ? { status: "ok", reply: "done" } : null,+ };+ },+ subscribeRun(_runId, listener, onResync) {+ return bus.subscribe(listener, { onResync });+ },+ syncRunStream() {},+ async getRunToolEntries(_runId, viewer, afterSeq) {+ viewers.push(viewer);+ const rows = entries.filter((entry) => afterSeq === undefined || entry.seq > afterSeq);+ reads.push({ afterSeq, rows: rows.length });+ return rows;+ },+ } as Pick<App, "getRun" | "getRunToolEntries" | "subscribeRun" | "syncRunStream">;+ const server = createServer((req, res) => {+ void runEventRoutes[0]!.handle({+ app,+ req,+ res,+ params: { id: "run" },+ actor: { p: "U1" },+ } as unknown as ApiCtx);+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const controller = new AbortController();+ const response = await fetch(`http://127.0.0.1:${(server.address() as AddressInfo).port}`, {+ signal: controller.signal,+ });+ const reader = response.body!.getReader();+ let wire = "";+ const consume = (async () => {+ while (true) {+ const next = await reader.read();+ if (next.done) return;+ wire += new TextDecoder().decode(next.value);+ }+ })();+ return {+ bus,+ viewers,+ reads,+ consume,+ wire: () => wire,+ async close() {+ controller.abort();+ await consume.catch(() => undefined);+ server.closeAllConnections();+ await new Promise<void>((resolve) => server.close(() => resolve()));+ },+ };+}++const lookup = toolEntry(1, "tool_call", {+ tool: "salesforce_query",+ mcpServer: "salesforce",+ args: { q: "acme" },+ callId: "c1",+});+const lookupResult = toolEntry(2, "tool_result", {+ tool: "salesforce_query",+ mcpServer: "salesforce",+ callId: "c1",+ isError: false,+ result: "1 account",+});++test("run events replay a run's recorded tool calls and results on first connect", async () => {+ const stream = await streamToolRun([lookup, lookupResult], { status: "running" });+ try {+ await sleep(50);+ assert.deepEqual(toolFrames(stream.wire()), [+ {+ id: "tool:c1:start",+ data: {+ type: "TOOL_CALL_START",+ toolCallId: "c1",+ toolCallName: "salesforce_query",+ args: { mcpServer: "salesforce", args: { q: "acme" } },+ },+ },+ {+ id: "tool:c1:result",+ data: { type: "TOOL_CALL_RESULT", toolCallId: "c1", content: "1 account", isError: false },+ },+ ]);+ assert.ok(stream.viewers.length > 0 && stream.viewers.every((viewer) => viewer === "U1"));+ } finally {+ await stream.close();+ }+});++test("run events forward tool entries recorded after connect on refresh, exactly once each", async () => {+ const entries = [lookup];+ const stream = await streamToolRun(entries, { status: "running" });+ try {+ await sleep(50);+ assert.deepEqual(+ toolFrames(stream.wire()).map((frame) => frame.id),+ ["tool:c1:start"],+ );+ entries.push(lookupResult, toolEntry(3, "tool_call", { tool: "execute", command: "ls", callId: "c2" }));+ stream.bus.emit({ runId: "run", kind: "refresh" });+ await sleep(50);+ stream.bus.emit({ runId: "run", kind: "refresh" });+ stream.bus.emit({ runId: "run", kind: "refresh" });+ await sleep(50);+ entries.push(toolEntry(4, "tool_result", { tool: "execute", callId: "c2", isError: true, result: "denied" }));+ stream.bus.emit({ runId: "run", kind: "refresh" });+ await sleep(50);+ assert.deepEqual(+ toolFrames(stream.wire()).map((frame) => frame.id),+ ["tool:c1:start", "tool:c1:result", "tool:c2:start", "tool:c2:result"],+ );+ assert.deepEqual(toolFrames(stream.wire())[3]!.data, {+ type: "TOOL_CALL_RESULT",+ toolCallId: "c2",+ content: "denied",+ isError: true,+ });+ } finally {+ await stream.close();+ }+});++test("a finished run replays its tool events and then RUN_FINISHED", async () => {+ const stream = await streamToolRun([lookup, lookupResult], { status: "done" });+ try {+ await stream.consume;+ assert.deepEqual(+ frames(stream.wire()).map(+ (frame) => frame.id ?? `${frame.data.type}${frame.data.name ? `:${frame.data.name}` : ""}`,+ ),+ ["RUN_STARTED", "tool:c1:start", "tool:c1:result", "CUSTOM:run", "done"],+ );+ assert.equal(stream.bus.size(), 0);+ } finally {+ await stream.close();+ }+});++test(+ "a tool replay larger than the send buffer drains instead of dropping the connection",+ { timeout: 10_000 },+ async () => {+ const bulky = "x".repeat(100_000);+ const entries = Array.from({ length: 30 }, (_, i) => [+ toolEntry(i * 2 + 1, "tool_call", { tool: "execute", command: `step ${i}`, callId: `c${i}` }),+ toolEntry(i * 2 + 2, "tool_result", { tool: "execute", callId: `c${i}`, isError: false, result: bulky }),+ ]).flat();+ const stream = await streamToolRun(entries, { status: "done" });+ try {+ await stream.consume;+ assert.equal(toolFrames(stream.wire()).length, 60);+ assert.equal(frames(stream.wire()).at(-1)?.id, "done");+ } finally {+ await stream.close();+ }+ },+);++test("refreshes read only tool entries recorded after the last one sent", async () => {+ const entries = [lookup, lookupResult];+ const stream = await streamToolRun(entries, { status: "running" });+ try {+ await sleep(50);+ entries.push(toolEntry(3, "tool_call", { tool: "execute", command: "ls", callId: "c2" }));+ stream.bus.emit({ runId: "run", kind: "refresh" });+ await sleep(50);+ stream.bus.emit({ runId: "run", kind: "refresh" });+ await sleep(50);+ assert.deepEqual(stream.reads, [+ { afterSeq: undefined, rows: 2 },+ { afterSeq: 2, rows: 0 },+ { afterSeq: 2, rows: 1 },+ { afterSeq: 3, rows: 0 },+ ]);+ } finally {+ await stream.close();+ }+});++test(+ "a final tool event larger than the send buffer is still followed by RUN_FINISHED",+ { timeout: 10_000 },+ async () => {+ const stream = await streamToolRun(+ [+ lookup,+ toolEntry(2, "tool_result", {+ tool: "salesforce_query",+ callId: "c1",+ isError: false,+ result: "é".repeat(1_500_000),+ }),+ ],+ { status: "done" },+ );+ try {+ await stream.consume;+ assert.deepEqual(+ frames(stream.wire()).map((frame) => frame.id ?? frame.data.type),+ ["RUN_STARTED", "tool:c1:start", "tool:c1:result", "CUSTOM", "done"],+ );+ } finally {+ await stream.close();+ }+ },+);diff --git a/test/run-tool-entries.test.ts b/test/run-tool-entries.test.tsnew file mode 100644index 000000000..5cc2cb473--- /dev/null+++ b/test/run-tool-entries.test.ts@@ -0,0 +1,129 @@+import "./support/auto-fake-sprites.ts";++import { test } from "node:test";+import assert from "node:assert/strict";+import { mkdtempSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { buildApp } from "../src/wiring.ts";+import type { TurnRequest } from "../src/types.ts";+import { testConfig } from "./support/test-config.ts";++const actor = { externalId: "U1" };+function dm(text: string): TurnRequest {+ return {+ surface: "slack",+ actor,+ conversation: { kind: "dm", threadRef: "dm:D-tools", audience: [actor] },+ deliveryTarget: "slack:D-tools",+ text,+ liveActor: true,+ async: true,+ };+}++async function settled(built: ReturnType<typeof buildApp>, runId: string) {+ for (let i = 0; i < 300; i++) {+ const run = await built.runs.get(runId);+ if (run && (run.status === "done" || run.status === "failed")) return run;+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ throw new Error(`run ${runId} did not settle`);+}++test("a run's tool entries are read from the session tape, bounded by its turn and its viewer", async () => {+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "run-tools-")) }));+ built.runtime.start();+ try {+ const first = await built.app.turn(dm("!run echo hi"));+ const run = await settled(built, first.runId!);+ const second = await built.app.turn(dm("plain follow-up"));+ await settled(built, second.runId!);+ const owner = run.request.actor.id;+ const entries = await built.app.getRunToolEntries(first.runId!, owner);+ assert.deepEqual(+ entries.map((e) => e.type),+ ["tool_call", "tool_result"],+ );+ assert.equal((entries[0]!.payload as { command?: string }).command, "echo hi");+ assert.ok(entries.every((e) => e.seq > run.turnUserSeq!));+ assert.deepEqual(await built.app.getRunToolEntries(second.runId!, owner), []);+ assert.deepEqual(await built.app.getRunToolEntries(first.runId!, "internal:someone-else"), []);+ assert.deepEqual(await built.app.getRunToolEntries("missing-run", owner), []);+ } finally {+ await built.runtime.stop();+ }+});++test("a run's tool entries include rounds after a mid-run goal prompt and stop at the next run", async () => {+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "run-tools-goal-")) }));+ built.runtime.start();+ try {+ const first = await built.app.turn(dm("!run echo hi"));+ const run = await settled(built, first.runId!);+ const session = (await built.sessions.getByThread(run.sessionId))!;+ const { lease } = await built.sessions.acquireLease(session.id);+ const scopeLabel = session.scopeId;+ await built.sessions.append(lease!, {+ type: "user",+ payload: {+ text: "[goal] The active goal is not marked complete. Continue working toward it.",+ hidden: true,+ runId: run.id,+ },+ scopeLabel,+ });+ const call = await built.sessions.append(lease!, {+ type: "tool_call",+ payload: { tool: "execute", command: "make test", callId: "goal-round" },+ scopeLabel,+ });+ await built.sessions.append(lease!, {+ type: "tool_result",+ payload: { tool: "execute", callId: "goal-round", isError: false, result: "ok" },+ scopeLabel,+ });+ await built.sessions.releaseLease(lease!);+ const second = await built.app.turn(dm("!run echo again"));+ await settled(built, second.runId!);+ const owner = run.request.actor.id;+ const callIds = (entries: Array<{ payload: unknown }>) =>+ entries.map((e) => (e.payload as { callId?: string }).callId);+ const all = await built.app.getRunToolEntries(run.id, owner);+ assert.equal(all.length, 4);+ assert.deepEqual(callIds(all.slice(2)), ["goal-round", "goal-round"]);+ assert.deepEqual(callIds(await built.app.getRunToolEntries(run.id, owner, call.seq)), ["goal-round"]);+ const next = await built.app.getRunToolEntries(second.runId!, owner);+ assert.equal(next.length, 2);+ assert.ok(next.every((e) => (e.payload as { callId?: string }).callId !== "goal-round"));+ } finally {+ await built.runtime.stop();+ }+});++test("each tool entry nudges the run stream once it is durable", async () => {+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "run-tools-nudge-")) }));+ const log: string[] = [];+ const append = built.sessions.append.bind(built.sessions);+ built.sessions.append = async (lease, entry) => {+ const appended = await append(lease, entry);+ log.push(appended.type);+ return appended;+ };+ try {+ const turn = await built.app.turn(dm("!run echo hi"));+ const unsubscribe = built.app.subscribeRun(turn.runId!, (event) => {+ if (event.kind === "refresh") log.push("refresh");+ });+ built.runtime.start();+ await settled(built, turn.runId!);+ unsubscribe();+ for (const type of ["tool_call", "tool_result"]) {+ const at = log.indexOf(type);+ assert.ok(at >= 0, `${type} was appended`);+ assert.equal(log[at + 1], "refresh", `${type} is followed by a refresh before anything else is appended`);+ }+ } finally {+ await built.runtime.stop();+ }+});