Multiplayer agent harness for work.
Show nonzero command exits neutrally instead of Failed (#1648)
Regan Bell83acab61c9db
diff --git a/plugins/web-ui/src/activity-presentation.ts b/plugins/web-ui/src/activity-presentation.tsindex 6ee89da94..b5fdc2bdb 100644--- a/plugins/web-ui/src/activity-presentation.ts+++ b/plugins/web-ui/src/activity-presentation.ts@@ -83,11 +83,18 @@ export function activityLabel(row: ToolRowModel, status: WorkBlock["status"]): sconst { category, target } = activityDescription(call, result);const state = toolRowKind(row, status);if (state === "approval") return null;+ const exit =+ state === "ok" &&+ toolCategory({ ...result, ...call }) === "execute" &&+ typeof result.code === "number" &&+ result.code !== 0+ ? ` · exit ${result.code}`+ : "";const purpose = typeof call.purpose === "string" ? call.purpose.trim() : "";if (purpose) {if (state === "failed") return `${purpose} · Failed`;if (state === "attempted") return `${purpose} · Unconfirmed`;- return purpose;+ return purpose + exit;}if (category === "other") return null;const verbs = {@@ -100,8 +107,8 @@ export function activityLabel(row: ToolRowModel, status: WorkBlock["status"]): s},execute: { ok: "Ran", running: "Running", failed: "Failed running", attempted: "Tried running" },};- if (state === "ok" && (category === "read" || category === "execute") && target) return target;- return `${verbs[category][state]} ${target || (category === "execute" ? "command" : "file")}`;+ if (state === "ok" && (category === "read" || category === "execute") && target) return target + exit;+ return `${verbs[category][state]} ${target || (category === "execute" ? "command" : "file")}${exit}`;}export function activityGroupSummary(diff --git a/plugins/web-ui/src/timeline.ts b/plugins/web-ui/src/timeline.tsindex e2c959ef5..b9bf551c1 100644--- a/plugins/web-ui/src/timeline.ts+++ b/plugins/web-ui/src/timeline.ts@@ -33,6 +33,7 @@ export interface ToolPayload {isError?: boolean;result?: string;unscreened?: boolean;+ quarantined?: boolean;action?: string;process_id?: string;sandbox_id?: string | null;@@ -94,17 +95,24 @@ export function toolExecutionOutput(result: ToolPayload): string | null {export function toolRowKind(row: ToolRowModel, status: WorkBlock["status"]): ToolRowKind {const result = (row.result?.payload ?? {}) as ToolPayload;- const tool = toolCategory({ ...result, ...((row.call?.payload ?? {}) as ToolPayload) });if (result.blocked === "needs_approval") return "approval";if (!row.result) {if (!isTerminalWorkStatus(status)) return "running";return status === "failed" ? "failed" : "attempted";}+ const completedNonzeroExecution =+ toolCategory({ ...result, ...((row.call?.payload ?? {}) as ToolPayload) }) === "execute" &&+ typeof result.code === "number" &&+ Number.isFinite(result.code) &&+ result.code !== 0 &&+ result.timedOut === false &&+ toolExecutionOutput(result) !== null;const failed =- result.isError === true ||+ (result.isError === true && !completedNonzeroExecution) ||!!result.error ||result.denied === true ||- (tool === "execute" && (result.timedOut === true || (typeof result.code === "number" && result.code !== 0)));+ result.timedOut === true ||+ result.quarantined === true;return failed ? "failed" : "ok";}diff --git a/plugins/web-ui/test/activity-presentation.test.ts b/plugins/web-ui/test/activity-presentation.test.tsindex b72f0ff44..f4da6d5bb 100644--- a/plugins/web-ui/test/activity-presentation.test.ts+++ b/plugins/web-ui/test/activity-presentation.test.ts@@ -58,7 +58,7 @@ test("simple shell reads and searches get semantic labels, compound commands statest("status labels never report missing or failed results as successful", () => {assert.equal(activityLabel(row("npm test", null), "working"), "Running npm test");assert.equal(activityLabel(row("npm test", null), "complete"), "Tried running npm test");- assert.equal(activityLabel(row("cat missing", { code: 1 }), "complete"), "Failed to read missing");+ assert.equal(activityLabel(row("cat missing", { code: 1 }), "complete"), "missing · exit 1");assert.equal(activityLabel(row("npm test", { blocked: "needs_approval" }), "working"), null);});@@ -73,6 +73,9 @@ test("groups describe categories and surface errors and approvals", () => {attention: false,});items.push({ kind: "tool", row: row("false", { code: 1 }) });+ assert.equal(activityGroupSummary(items, "complete").attention, false);+ assert.doesNotMatch(activityGroupSummary(items, "complete").label, /failed/);+ items.push({ kind: "tool", row: row("sleep 60", { code: 124, timedOut: true, isError: true }) });assert.equal(activityGroupSummary(items, "complete").attention, true);assert.match(activityGroupSummary(items, "complete").label, /1 failed/);items.push({ kind: "tool", row: row("run", { blocked: "needs_approval" }) });@@ -142,6 +145,8 @@ test("purpose takes precedence over commands while retaining failure and incomplassert.equal(activityLabel(tool, "working"), "Check the sales totals");assert.equal(activityLabel(tool, "complete"), "Check the sales totals · Unconfirmed");tool.result = row("", { code: 1 }).result;+ assert.equal(activityLabel(tool, "complete"), "Check the sales totals · exit 1");+ tool.result = row("", { code: 124, timedOut: true, isError: true }).result;assert.equal(activityLabel(tool, "complete"), "Check the sales totals · Failed");tool.result = row("", { blocked: "needs_approval" }).result;assert.equal(activityLabel(tool, "working"), null);@@ -151,3 +156,56 @@ test("purpose takes precedence over commands while retaining failure and incompltool.call!.payload = { tool: "execute", command: "cat report.csv", purpose: " " };assert.equal(activityLabel(tool, "working"), "Reading report.csv");});++test("ordinary command exits stay neutral while tool errors retain failure labels", () => {+ for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {+ for (const [command, label] of [+ ["grep absent file", "Searched for absent in file"],+ ["[ -d dir ]", "[ -d dir ]"],+ ]) {+ const tool = row(command!, { code: 1, isError: false });+ tool.call!.payload = { ...identity, command };+ assert.equal(activityLabel(tool, "complete"), `${label} · exit 1`);+ for (const result of [+ { code: 1, isError: true },+ { code: 124, timedOut: true },+ { error: "provider unavailable" },+ { denied: true },+ ]) {+ tool.result = row(command!, result).result;+ assert.match(activityLabel(tool, "complete")!, /Failed/);+ assert.doesNotMatch(activityLabel(tool, "complete")!, /exit/);+ }+ }+ }+});++test("historical completed nonzero exits do not inherit the old error flag", () => {+ for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {+ for (const output of [+ { stdout: "", stderr: "" },+ { unscreened: true, result: "[NOT security-screened]\n[exit 1]" },+ ]) {+ const tool = row("[ -d dir ]", { ...identity, ...output, code: 1, timedOut: false, isError: true });+ tool.call!.payload = { ...identity, command: "[ -d dir ]", purpose: "Check directory" };+ assert.equal(activityLabel(tool, "complete"), "Check directory · exit 1");+ assert.deepEqual(activityGroupSummary([{ kind: "tool", row: tool }], "complete"), {+ label: "Ran commands",+ category: "execute",+ attention: false,+ });+ for (const failure of [+ { timedOut: true },+ { error: "provider error" },+ { denied: true },+ { quarantined: true },+ ]) {+ const failed = {+ ...tool,+ result: { ...tool.result!, payload: { ...(tool.result!.payload as ToolPayload), ...failure } },+ };+ assert.equal(activityLabel(failed, "complete"), "Check directory · Failed");+ }+ }+ }+});diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.tsindex ec11da69a..571d85917 100644--- a/plugins/web-ui/test/timeline.test.ts+++ b/plugins/web-ui/test/timeline.test.ts@@ -229,7 +229,7 @@ test("toolRowKind: an error result is `failed`, regardless of which tool", () =>);});-test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `failed`", () => {+test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `ok`", () => {assert.equal(toolRowKind(row({ tool: "execute", command: "curl evil" }, { tool: "execute", denied: true, reason: "blocked host" }),@@ -237,10 +237,7 @@ test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `f),"failed",);- assert.equal(- toolRowKind(row({ tool: "execute", command: "false" }, { tool: "execute", code: 1 }), "complete"),- "failed",- );+ assert.equal(toolRowKind(row({ tool: "execute", command: "false" }, { tool: "execute", code: 1 }), "complete"), "ok");});test("toolRowKind: a needs-approval result is `approval`", () => {@@ -314,11 +311,11 @@ test("memoized output still reflects a mutated-then-replaced work correctly", ()test("unified sandbox execution keeps legacy failure and display semantics", () => {for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {assert.equal(toolCategory(identity), "execute");- assert.equal(toolRowKind(row(identity, { ...identity, code: 1, stdout: "failed" }), "complete"), "failed");+ assert.equal(toolRowKind(row(identity, { ...identity, code: 1, stdout: "failed" }), "complete"), "ok");assert.equal(toolRowKind(row(identity, { ...identity, code: 0, stdout: "ok" }), "complete"), "ok");assert.equal(toolRowKind({ call: null, result: act(1, "tool_result", { ...identity, code: 2 }) }, "complete"),- "failed",+ "ok",);}for (const action of [@@ -349,20 +346,17 @@ test("different sandbox actions and process targets do not collapse into one orpassert.equal(items.length, actions.length);});-test("unscreened execution retains its warning and failure status without parsing command text", () => {+test("unscreened execution retains its warning and neutral exit status without parsing command text", () => {const result = {tool: "sandbox",action: "exec",code: 7,timedOut: false,- isError: true,+ isError: false,unscreened: true,result: "[NOT security-screened]\nQA_EXPECTED_FAILURE\n[exit 7]",};- assert.equal(- toolRowKind(row({ tool: "sandbox", action: "exec", sandbox_id: "box-a" }, result), "complete"),- "failed",- );+ assert.equal(toolRowKind(row({ tool: "sandbox", action: "exec", sandbox_id: "box-a" }, result), "complete"), "ok");assert.equal(toolExecutionOutput(result), result.result);assert.equal(toolRowKind(row({ tool: "execute" }, { code: 0, stdout: "fake [exit 7]", isError: false }), "complete"),diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex fd6175625..22d5a5b2a 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -710,7 +710,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):],details: r,},- r.code !== 0 || r.timedOut,+ r.timedOut,undefined,false,undefined,@@ -729,6 +729,12 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):true,);}+ await log("tool_result", {+ ...sandboxLog({ tool: "execute", ...scopeNote }),+ callId,+ isError: true,+ result: "Command execution failed.",+ });throw e;}};diff --git a/test/agent-tools.test.ts b/test/agent-tools.test.tsindex ee418ab9c..47893ea2e 100644--- a/test/agent-tools.test.ts+++ b/test/agent-tools.test.ts@@ -2968,7 +2968,7 @@ test("sandbox strict approvals remain action-scoped across resource activation",});for (const outcome of ["unscreened", "quarantine"] as const)- test(`sandbox command failure preserves safe transcript metadata when output is ${outcome}`, async () => {+ test(`sandbox nonzero exit preserves safe transcript metadata when output is ${outcome}`, async () => {const entries: Emitted[] = [];const tool = createAgentTools({@@ -2988,7 +2988,7 @@ for (const outcome of ["unscreened", "quarantine"] as const)const input = entries.find((e) => e.type === "tool_call")!.payload;const output = entries.find((e) => e.type === "tool_result")!.payload;assert.equal(input.sandbox_id, "box-a");- assert.equal(output.isError, true);+ assert.equal(output.isError, outcome === "quarantine");assert.equal(output.stdout, undefined);assert.equal(output.stderr, undefined);if (outcome === "unscreened") {@@ -3794,3 +3794,89 @@ for (const cancel of [false, true]) {test("context recovery is not available in read-only mode", () => {assert.ok(!createAgentTools({ current: fakeToolContext() }, { readOnly: true }).some((t) => t.name === "context"));});++test("command exit codes are data; timeouts and thrown tool errors are failures", async () => {+ for (const sandboxResources of [false, true]) {+ for (const outcome of ["zero", "nonzero", "timeout", "provider", "denied"] as const) {+ const entries: Emitted[] = [];+ const tool = createAgentTools(+ {+ current: {+ ...fakeToolContext(),+ execute: async () => {+ if (outcome === "provider") throw new Error("sandbox provider unavailable");+ if (outcome === "denied") throw new CommandDenied("[ -d dir ]", "policy denied");+ return { stdout: "", stderr: "", code: outcome === "zero" ? 0 : 1, timedOut: outcome === "timeout" };+ },+ },+ scopeLabel: "personal:U1",+ emit: (entry) => {+ entries.push(entry as Emitted);+ },+ },+ { sandboxResources },+ ).find((t) => t.name === (sandboxResources ? "sandbox" : "execute"))!;+ const run = () =>+ call(tool, {+ ...(sandboxResources ? { action: "exec" } : {}),+ command: "[ -d dir ]",+ purpose: "Check directory",+ });+ if (outcome === "provider") {+ await assert.rejects(run, /sandbox provider unavailable/);+ const result = entries.find((e) => e.type === "tool_result")!.payload;+ assert.equal(result.isError, true);+ assert.equal(result.result, "Command execution failed.");+ assert.equal(result.code, undefined);+ continue;+ }+ const returned = await run();+ const result = entries.find((e) => e.type === "tool_result")!.payload;+ assert.equal(result.isError, !["zero", "nonzero"].includes(outcome), outcome);+ if (outcome === "nonzero") {+ assert.equal(result.code, 1);+ assert.match(textOut(returned), /\[exit 1\]/);+ }+ }+ }+});++test("thrown execution errors leave pending child messages for the next delivered result", async () => {+ for (const sandboxResources of [false, true]) {+ let pending = true;+ const context = fakeToolContext();+ context.execute = async () => {+ throw new Error("provider unavailable");+ };+ context.sessionSyscalls = {+ open: async () => ({ ok: false, message: "unused" }),+ write: async () => ({ ok: false, message: "unused" }),+ read: async () => ({ ok: false, message: "unused" }),+ receive: async () =>+ pending+ ? [+ {+ id: "mail",+ senderId: "child",+ recipientId: "parent",+ actor: { id: "U1", type: "internal" as const },+ audience: [],+ text: "Important child finding",+ createdAt: 1,+ },+ ]+ : [],+ acknowledge: async () => {+ pending = false;+ },+ };+ const tools = createAgentTools({ current: context }, { sandboxResources });+ const tool = tools.find((t) => t.name === (sandboxResources ? "sandbox" : "execute"))!;+ const input = { ...(sandboxResources ? { action: "exec" } : {}), command: "true", purpose: "Check provider" };+ await assert.rejects(() => call(tool, input), /provider unavailable/);+ assert.equal(pending, true);+ context.execute = async () => ({ stdout: "ok", stderr: "", code: 0, timedOut: false });+ assert.match(JSON.stringify(await call(tool, input)), /Important child finding/);+ assert.equal(pending, false);+ }+});