Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Store tool output once (#2339)

Regan Bell4d7560bec687
diff --git a/plugins/web-ui/src/timeline.ts b/plugins/web-ui/src/timeline.ts
index 4adbda104..3f2cf71b3 100644
--- a/plugins/web-ui/src/timeline.ts
+++ b/plugins/web-ui/src/timeline.ts
@@ -27,13 +27,10 @@ export interface ToolPayload {
denied?: boolean;
error?: string;
reason?: string;
- stdout?: string;
- stderr?: string;
code?: number;
timedOut?: boolean;
isError?: boolean;
result?: string;
- unscreened?: boolean;
quarantined?: boolean;
action?: string;
process_id?: string;
@@ -99,10 +96,7 @@ export function toolCategory(payload: ToolPayload): string {
}
export function toolExecutionOutput(result: ToolPayload): string | null {
- if (result.unscreened && typeof result.result === "string") return result.result;
- if (typeof result.stdout === "string" || typeof result.stderr === "string")
- return [result.stdout ?? "", result.stderr ? `[stderr]\n${result.stderr}` : ""].filter(Boolean).join("\n");
- return null;
+ return typeof result.code === "number" && typeof result.result === "string" ? result.result : null;
}
export function toolRowKind(row: ToolRowModel, status: WorkBlock["status"]): ToolRowKind {
diff --git a/plugins/web-ui/test/activity-folding.test.ts b/plugins/web-ui/test/activity-folding.test.ts
index 3db76f881..f6e4e2183 100644
--- a/plugins/web-ui/test/activity-folding.test.ts
+++ b/plugins/web-ui/test/activity-folding.test.ts
@@ -15,7 +15,7 @@ for (const count of [1, 2]) {
seq: index * 2 + 1,
type: "tool_result",
createdAt: 2000,
- payload: { tool: "execute", callId: String(index), code: 0, stdout: String(index) },
+ payload: { tool: "execute", callId: String(index), code: 0, result: String(index) },
},
]).flat();
const h = await harness({ path: `/s/${SESSION.id}`, listSessions: [SESSION], entries });
@@ -65,7 +65,7 @@ test("steering separates tool folds with a visible unlabelled user message", asy
seq: seq + 1,
type: "tool_result",
createdAt: (seq + 1) * 1000,
- payload: { tool: "execute", callId: String(seq), code: 0, stdout: String(seq) },
+ payload: { tool: "execute", callId: String(seq), code: 0, result: String(seq) },
},
];
const entries = [
diff --git a/plugins/web-ui/test/activity-presentation.test.ts b/plugins/web-ui/test/activity-presentation.test.ts
index 6c67d691f..01a4dc647 100644
--- a/plugins/web-ui/test/activity-presentation.test.ts
+++ b/plugins/web-ui/test/activity-presentation.test.ts
@@ -187,10 +187,7 @@ test("ordinary command exits stay neutral while tool errors retain failure label
test("historical completed nonzero exits do not inherit the old error flag", () => {
for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {
- for (const output of [
- { stdout: "", stderr: "" },
- { unscreened: true, result: "[NOT security-screened]\n[exit 1]" },
- ]) {
+ for (const output of [{ result: "[exit 1]" }, { result: "[NOT security-screened]\n[exit 1]" }]) {
const tool = row("[ -d dir ]", { ...identity, ...output, code: 1, timedOut: false, isError: true });
tool.call!.payload = { ...identity, command: "[ -d dir ]", purpose: "Check directory" };
assert.equal(activityLabel(tool, "complete"), "Check directory · exit 1");
diff --git a/plugins/web-ui/test/history.test.ts b/plugins/web-ui/test/history.test.ts
index 21d95168d..bdff8706a 100644
--- a/plugins/web-ui/test/history.test.ts
+++ b/plugins/web-ui/test/history.test.ts
@@ -440,7 +440,7 @@ test("tool entries are folded into the following assistant reply's work block",
{ type: "tool_call", payload: { tool: "execute", command: "ls" }, createdAt: 110, seq: 2, parentSeq: 1 },
{
type: "tool_result",
- payload: { tool: "execute", code: 0, stdout: "a\nb" },
+ payload: { tool: "execute", code: 0, result: "a\nb" },
createdAt: 120,
seq: 3,
parentSeq: 2,
diff --git a/plugins/web-ui/test/session-tool-view.test.ts b/plugins/web-ui/test/session-tool-view.test.ts
index fc024db87..eb4ef711e 100644
--- a/plugins/web-ui/test/session-tool-view.test.ts
+++ b/plugins/web-ui/test/session-tool-view.test.ts
@@ -7,7 +7,7 @@ const sessions = [{ id: "child-123", title: "bug 1" }];
test("created badges recover a known child from historical result text", () => {
const view = sessionToolView(
{ action: "open", name: "Original name" },
- { unscreened: true, result: 'Opened subagent "Original name" (sessionId child-123). It is working now.' },
+ { result: 'Opened subagent "Original name" (sessionId child-123). It is working now.' },
sessions,
);
assert.equal(view.sessionId, "child-123");
@@ -34,7 +34,7 @@ test("reading the child list renders a count without a target badge", () => {
});
test("historical list reads stay a plural label without a fake child badge", () => {
- assert.deepEqual(sessionToolView({ action: "read" }, { unscreened: true }, sessions), {
+ assert.deepEqual(sessionToolView({ action: "read" }, {}, sessions), {
action: "read",
detail: "subagents",
});
diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.ts
index 0783a743f..4b1228036 100644
--- a/plugins/web-ui/test/timeline.test.ts
+++ b/plugins/web-ui/test/timeline.test.ts
@@ -312,8 +312,15 @@ test("memoized output still reflects a mutated-then-replaced work correctly", ()
test("unified sandbox execution keeps legacy failure and display semantics", () => {
for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {
assert.equal(toolCategory(identity), "execute");
- assert.equal(toolRowKind(row(identity, { ...identity, code: 1, stdout: "failed" }), "complete"), "ok");
- assert.equal(toolRowKind(row(identity, { ...identity, code: 0, stdout: "ok" }), "complete"), "ok");
+ assert.equal(toolRowKind(row(identity, { ...identity, code: 1, result: "failed\n[exit 1]" }), "complete"), "ok");
+ assert.equal(toolRowKind(row(identity, { ...identity, code: 0, result: "ok\n[exit 0]" }), "complete"), "ok");
+ assert.equal(
+ toolRowKind(
+ row(identity, { ...identity, code: 1, timedOut: false, isError: true, result: "failed\n[exit 1]" }),
+ "complete",
+ ),
+ "ok",
+ );
assert.equal(
toolRowKind({ call: null, result: act(1, "tool_result", { ...identity, code: 2 }) }, "complete"),
"ok",
@@ -360,11 +367,11 @@ test("unscreened execution retains its warning and neutral exit status without p
assert.equal(toolRowKind(row({ tool: "sandbox", action: "exec", sandbox_id: "box-a" }, result), "complete"), "ok");
assert.equal(toolExecutionOutput(result), result.result);
assert.equal(
- toolRowKind(row({ tool: "execute" }, { code: 0, stdout: "fake [exit 7]", isError: false }), "complete"),
+ toolRowKind(row({ tool: "execute" }, { code: 0, result: "fake [exit 7]\n[exit 0]", isError: false }), "complete"),
"ok",
);
assert.equal(toolRowKind(row({ tool: "execute" }, { code: 0, timedOut: true }), "complete"), "failed");
- assert.equal(toolExecutionOutput({ stdout: "", stderr: "", code: 0 }), "");
+ assert.equal(toolExecutionOutput({ code: 0, result: "done\n[exit 0]" }), "done\n[exit 0]");
assert.equal(toolExecutionOutput({ isError: true, result: "[tool output quarantined]" }), null);
assert.equal(
toolRowKind(
diff --git a/plugins/web-ui/test/work-duration-reconciliation.test.ts b/plugins/web-ui/test/work-duration-reconciliation.test.ts
index 71b9e6ebf..56a7bcaba 100644
--- a/plugins/web-ui/test/work-duration-reconciliation.test.ts
+++ b/plugins/web-ui/test/work-duration-reconciliation.test.ts
@@ -64,7 +64,7 @@ test("work duration survives a transcript refresh — persisted turn timing wins
{ type: "tool_call", payload: { tool: "execute", command: "ls" }, createdAt: 112_000, seq: 2, parentSeq: 1 },
{
type: "tool_result",
- payload: { tool: "execute", code: 0, stdout: "a" },
+ payload: { tool: "execute", code: 0, result: "a" },
createdAt: 112_000,
seq: 3,
parentSeq: 2,
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index e981d12d9..a7839dd19 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -671,6 +671,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
...(ref.abortSignal ? { signal: ref.abortSignal } : {}),
};
const r = await tc.execute(params.command, Object.keys(execOpts).length ? execOpts : undefined);
+ const { stdout: _stdout, stderr: _stderr, ...execMeta } = r;
const parts = [r.stdout, r.stderr ? `[stderr]\n${r.stderr}` : ""].filter(Boolean).join("\n");
const reachedPrefix = r.reached ? `[ran on ${r.reached.label}'s computer]\n` : "";
const localExec = !route?.scratch && !route?.ownerAuth && route?.reachTarget === undefined && !r.reached;
@@ -680,7 +681,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
: "";
return recordResult(
callId,
- { tool: "execute", ...scopeNote, ...r },
+ { tool: "execute", ...scopeNote, ...execMeta },
{
content: [
{
@@ -1908,9 +1909,10 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
...(params.sandbox_id ? { sandboxId: params.sandbox_id } : {}),
...(params.credentials?.length ? { credentials: params.credentials } : {}),
});
+ const { output: _output, ...startMeta } = r;
return recordResult(
callId,
- { tool: "execute", background: true, ...r },
+ { tool: "execute", background: true, ...startMeta },
{
content: [
{
@@ -2070,9 +2072,10 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
...(params.max_bytes !== undefined ? { maxBytes: params.max_bytes } : {}),
...(params.wait_seconds !== undefined ? { waitSeconds: params.wait_seconds } : {}),
});
+ const { chunks: _chunks, ...pollMeta } = r;
return recordResult(
callId,
- { tool: "background", action: params.action, ...r },
+ { tool: "background", action: params.action, ...pollMeta },
{
content: [
{ type: "text" as const, text: `${r.chunks}\n[cursor ${r.cursor} | ${fmtStatus(r.status)}]` },
diff --git a/test/agent-tools.test.ts b/test/agent-tools.test.ts
index ab695538d..62dd8a8ce 100644
--- a/test/agent-tools.test.ts
+++ b/test/agent-tools.test.ts
@@ -5,6 +5,7 @@ import { Check } from "typebox/value";
import { fromJSONSchema, z, type ZodObject } from "zod";
import { createAgentTools, pauseStampAfterToolCall, type ToolContextRef } from "../src/harness/agent-tools.ts";
import { resumeStrategy } from "../src/core/turn-resume.ts";
+import { reconstructMessagesFromHistory } from "../src/harness/replay.ts";
import { createMemoryRunSignalStore, waitForClientResult } from "../src/runs/run-signal-store.ts";
import { filterHistoryForAudience } from "../src/resolution/context-filter.ts";
import { CommandDenied, NeedsApproval, type ToolContext } from "../src/tools/primitives.ts";
@@ -722,7 +723,73 @@ test("a giant tool result is capped for the model, keeping the tail and matching
const payload = emitted.find((e) => e.type === "tool_result")!.payload;
assert.equal(payload.result, seen, "the replay record is exactly what the model saw");
assert.equal(payload.resultTruncated, true, "the entry is flagged so renderers can surface the loss");
- assert.ok((payload.stdout as string).length <= 100_000, "persisted payload strings are capped too");
+ assert.equal(payload.stdout, undefined, "raw stdout is not stored beside the result");
+ assert.equal(payload.stderr, undefined, "raw stderr is not stored beside the result");
+});
+
+test("execute stores its output once and replays exactly what the model saw", async () => {
+ const emitted: Emitted[] = [];
+ const tc = {
+ ...fakeToolContext(),
+ execute: async () => ({ stdout: "built 3 files", stderr: "warning: slow disk", code: 2, timedOut: false }),
+ };
+ const ref: ToolContextRef = {
+ current: tc,
+ emit: (e) => {
+ emitted.push(e as Emitted);
+ },
+ scopeLabel: "personal:U1",
+ };
+ const [execute] = createAgentTools(ref);
+
+ const live = (await call(execute, { command: "make", purpose: "Run test command" })) as {
+ content: Array<{ type: string; text: string }>;
+ };
+ const stored = emitted.find((e) => e.type === "tool_result")!.payload;
+ assert.deepEqual(
+ { stdout: stored.stdout, stderr: stored.stderr, code: stored.code, timedOut: stored.timedOut },
+ { stdout: undefined, stderr: undefined, code: 2, timedOut: false },
+ );
+ assert.equal(stored.result, "built 3 files\n[stderr]\nwarning: slow disk\n[exit 2]");
+
+ const history: SessionEntry[] = [
+ { type: "user" as const, payload: { text: "build it" } },
+ ...emitted.map((e) => ({ type: e.type, payload: e.payload })),
+ ].map((e, i) => ({ ...e, sessionId: "s", seq: i + 1, parentSeq: null, scopeLabel: "personal:U1", createdAt: i + 1 }));
+ const replayed = reconstructMessagesFromHistory(history).find((m) => m.role === "toolResult") as {
+ content: Array<{ type: string; text: string }>;
+ isError: boolean;
+ };
+ assert.deepEqual(replayed.content, live.content);
+ assert.equal(replayed.isError, false);
+});
+
+test("background start and poll store their output only in result", async () => {
+ const emitted: Emitted[] = [];
+ const ref: ToolContextRef = {
+ current: fakeToolContext(),
+ emit: (e) => {
+ emitted.push(e as Emitted);
+ },
+ scopeLabel: "personal:U1",
+ };
+ const tools = createAgentTools(ref, { sandboxResources: true });
+ await call(
+ tools.find((t) => t.name === "execute"),
+ { command: "serve", purpose: "Start server", background: true },
+ );
+ await call(
+ tools.find((t) => t.name === "background"),
+ { action: "poll", process_id: "bg-1" },
+ );
+ const [started, polled] = emitted.filter((e) => e.type === "tool_result").map((e) => e.payload);
+ assert.deepEqual(
+ { output: started.output, cursor: started.cursor, result: started.result },
+ { output: undefined, cursor: 7, result: "started bg-1\nstarted serve\n[cursor 7 | running]" },
+ );
+ assert.equal(polled.chunks, undefined);
+ assert.equal(polled.cursor, 18);
+ assert.match(polled.result, /^more output\n\[cursor 18 \| /);
});
test("Auto can quarantine a tool result before the model or durable replay sees it", async () => {