Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Show nonzero command exits neutrally instead of Failed (#1648)

Regan Bell83acab61c9db
diff --git a/plugins/web-ui/src/activity-presentation.ts b/plugins/web-ui/src/activity-presentation.ts
index 6ee89da94..b5fdc2bdb 100644
--- a/plugins/web-ui/src/activity-presentation.ts
+++ b/plugins/web-ui/src/activity-presentation.ts
@@ -83,11 +83,18 @@ export function activityLabel(row: ToolRowModel, status: WorkBlock["status"]): s
const { category, target } = activityDescription(call, result);
const state = toolRowKind(row, status);
if (state === "approval") return null;
+ const exit =
+ state === "ok" &&
+ toolCategory({ ...result, ...call }) === "execute" &&
+ typeof result.code === "number" &&
+ result.code !== 0
+ ? ` · exit ${result.code}`
+ : "";
const purpose = typeof call.purpose === "string" ? call.purpose.trim() : "";
if (purpose) {
if (state === "failed") return `${purpose} · Failed`;
if (state === "attempted") return `${purpose} · Unconfirmed`;
- return purpose;
+ return purpose + exit;
}
if (category === "other") return null;
const verbs = {
@@ -100,8 +107,8 @@ export function activityLabel(row: ToolRowModel, status: WorkBlock["status"]): s
},
execute: { ok: "Ran", running: "Running", failed: "Failed running", attempted: "Tried running" },
};
- if (state === "ok" && (category === "read" || category === "execute") && target) return target;
- return `${verbs[category][state]} ${target || (category === "execute" ? "command" : "file")}`;
+ if (state === "ok" && (category === "read" || category === "execute") && target) return target + exit;
+ return `${verbs[category][state]} ${target || (category === "execute" ? "command" : "file")}${exit}`;
}
export function activityGroupSummary(
diff --git a/plugins/web-ui/src/timeline.ts b/plugins/web-ui/src/timeline.ts
index e2c959ef5..b9bf551c1 100644
--- a/plugins/web-ui/src/timeline.ts
+++ b/plugins/web-ui/src/timeline.ts
@@ -33,6 +33,7 @@ export interface ToolPayload {
isError?: boolean;
result?: string;
unscreened?: boolean;
+ quarantined?: boolean;
action?: string;
process_id?: string;
sandbox_id?: string | null;
@@ -94,17 +95,24 @@ export function toolExecutionOutput(result: ToolPayload): string | null {
export function toolRowKind(row: ToolRowModel, status: WorkBlock["status"]): ToolRowKind {
const result = (row.result?.payload ?? {}) as ToolPayload;
- const tool = toolCategory({ ...result, ...((row.call?.payload ?? {}) as ToolPayload) });
if (result.blocked === "needs_approval") return "approval";
if (!row.result) {
if (!isTerminalWorkStatus(status)) return "running";
return status === "failed" ? "failed" : "attempted";
}
+ const completedNonzeroExecution =
+ toolCategory({ ...result, ...((row.call?.payload ?? {}) as ToolPayload) }) === "execute" &&
+ typeof result.code === "number" &&
+ Number.isFinite(result.code) &&
+ result.code !== 0 &&
+ result.timedOut === false &&
+ toolExecutionOutput(result) !== null;
const failed =
- result.isError === true ||
+ (result.isError === true && !completedNonzeroExecution) ||
!!result.error ||
result.denied === true ||
- (tool === "execute" && (result.timedOut === true || (typeof result.code === "number" && result.code !== 0)));
+ result.timedOut === true ||
+ result.quarantined === true;
return failed ? "failed" : "ok";
}
diff --git a/plugins/web-ui/test/activity-presentation.test.ts b/plugins/web-ui/test/activity-presentation.test.ts
index b72f0ff44..f4da6d5bb 100644
--- a/plugins/web-ui/test/activity-presentation.test.ts
+++ b/plugins/web-ui/test/activity-presentation.test.ts
@@ -58,7 +58,7 @@ test("simple shell reads and searches get semantic labels, compound commands sta
test("status labels never report missing or failed results as successful", () => {
assert.equal(activityLabel(row("npm test", null), "working"), "Running npm test");
assert.equal(activityLabel(row("npm test", null), "complete"), "Tried running npm test");
- assert.equal(activityLabel(row("cat missing", { code: 1 }), "complete"), "Failed to read missing");
+ assert.equal(activityLabel(row("cat missing", { code: 1 }), "complete"), "missing · exit 1");
assert.equal(activityLabel(row("npm test", { blocked: "needs_approval" }), "working"), null);
});
@@ -73,6 +73,9 @@ test("groups describe categories and surface errors and approvals", () => {
attention: false,
});
items.push({ kind: "tool", row: row("false", { code: 1 }) });
+ assert.equal(activityGroupSummary(items, "complete").attention, false);
+ assert.doesNotMatch(activityGroupSummary(items, "complete").label, /failed/);
+ items.push({ kind: "tool", row: row("sleep 60", { code: 124, timedOut: true, isError: true }) });
assert.equal(activityGroupSummary(items, "complete").attention, true);
assert.match(activityGroupSummary(items, "complete").label, /1 failed/);
items.push({ kind: "tool", row: row("run", { blocked: "needs_approval" }) });
@@ -142,6 +145,8 @@ test("purpose takes precedence over commands while retaining failure and incompl
assert.equal(activityLabel(tool, "working"), "Check the sales totals");
assert.equal(activityLabel(tool, "complete"), "Check the sales totals · Unconfirmed");
tool.result = row("", { code: 1 }).result;
+ assert.equal(activityLabel(tool, "complete"), "Check the sales totals · exit 1");
+ tool.result = row("", { code: 124, timedOut: true, isError: true }).result;
assert.equal(activityLabel(tool, "complete"), "Check the sales totals · Failed");
tool.result = row("", { blocked: "needs_approval" }).result;
assert.equal(activityLabel(tool, "working"), null);
@@ -151,3 +156,56 @@ test("purpose takes precedence over commands while retaining failure and incompl
tool.call!.payload = { tool: "execute", command: "cat report.csv", purpose: " " };
assert.equal(activityLabel(tool, "working"), "Reading report.csv");
});
+
+test("ordinary command exits stay neutral while tool errors retain failure labels", () => {
+ for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {
+ for (const [command, label] of [
+ ["grep absent file", "Searched for absent in file"],
+ ["[ -d dir ]", "[ -d dir ]"],
+ ]) {
+ const tool = row(command!, { code: 1, isError: false });
+ tool.call!.payload = { ...identity, command };
+ assert.equal(activityLabel(tool, "complete"), `${label} · exit 1`);
+ for (const result of [
+ { code: 1, isError: true },
+ { code: 124, timedOut: true },
+ { error: "provider unavailable" },
+ { denied: true },
+ ]) {
+ tool.result = row(command!, result).result;
+ assert.match(activityLabel(tool, "complete")!, /Failed/);
+ assert.doesNotMatch(activityLabel(tool, "complete")!, /exit/);
+ }
+ }
+ }
+});
+
+test("historical completed nonzero exits do not inherit the old error flag", () => {
+ for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {
+ for (const output of [
+ { stdout: "", stderr: "" },
+ { unscreened: true, result: "[NOT security-screened]\n[exit 1]" },
+ ]) {
+ const tool = row("[ -d dir ]", { ...identity, ...output, code: 1, timedOut: false, isError: true });
+ tool.call!.payload = { ...identity, command: "[ -d dir ]", purpose: "Check directory" };
+ assert.equal(activityLabel(tool, "complete"), "Check directory · exit 1");
+ assert.deepEqual(activityGroupSummary([{ kind: "tool", row: tool }], "complete"), {
+ label: "Ran commands",
+ category: "execute",
+ attention: false,
+ });
+ for (const failure of [
+ { timedOut: true },
+ { error: "provider error" },
+ { denied: true },
+ { quarantined: true },
+ ]) {
+ const failed = {
+ ...tool,
+ result: { ...tool.result!, payload: { ...(tool.result!.payload as ToolPayload), ...failure } },
+ };
+ assert.equal(activityLabel(failed, "complete"), "Check directory · Failed");
+ }
+ }
+ }
+});
diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.ts
index ec11da69a..571d85917 100644
--- a/plugins/web-ui/test/timeline.test.ts
+++ b/plugins/web-ui/test/timeline.test.ts
@@ -229,7 +229,7 @@ test("toolRowKind: an error result is `failed`, regardless of which tool", () =>
);
});
-test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `failed`", () => {
+test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `ok`", () => {
assert.equal(
toolRowKind(
row({ tool: "execute", command: "curl evil" }, { tool: "execute", denied: true, reason: "blocked host" }),
@@ -237,10 +237,7 @@ test("toolRowKind: a policy-denied command is `failed`, and a nonzero exit is `f
),
"failed",
);
- assert.equal(
- toolRowKind(row({ tool: "execute", command: "false" }, { tool: "execute", code: 1 }), "complete"),
- "failed",
- );
+ assert.equal(toolRowKind(row({ tool: "execute", command: "false" }, { tool: "execute", code: 1 }), "complete"), "ok");
});
test("toolRowKind: a needs-approval result is `approval`", () => {
@@ -314,11 +311,11 @@ test("memoized output still reflects a mutated-then-replaced work correctly", ()
test("unified sandbox execution keeps legacy failure and display semantics", () => {
for (const identity of [{ tool: "execute" }, { tool: "sandbox", action: "exec" }]) {
assert.equal(toolCategory(identity), "execute");
- assert.equal(toolRowKind(row(identity, { ...identity, code: 1, stdout: "failed" }), "complete"), "failed");
+ assert.equal(toolRowKind(row(identity, { ...identity, code: 1, stdout: "failed" }), "complete"), "ok");
assert.equal(toolRowKind(row(identity, { ...identity, code: 0, stdout: "ok" }), "complete"), "ok");
assert.equal(
toolRowKind({ call: null, result: act(1, "tool_result", { ...identity, code: 2 }) }, "complete"),
- "failed",
+ "ok",
);
}
for (const action of [
@@ -349,20 +346,17 @@ test("different sandbox actions and process targets do not collapse into one orp
assert.equal(items.length, actions.length);
});
-test("unscreened execution retains its warning and failure status without parsing command text", () => {
+test("unscreened execution retains its warning and neutral exit status without parsing command text", () => {
const result = {
tool: "sandbox",
action: "exec",
code: 7,
timedOut: false,
- isError: true,
+ isError: false,
unscreened: true,
result: "[NOT security-screened]\nQA_EXPECTED_FAILURE\n[exit 7]",
};
- assert.equal(
- toolRowKind(row({ tool: "sandbox", action: "exec", sandbox_id: "box-a" }, result), "complete"),
- "failed",
- );
+ assert.equal(toolRowKind(row({ tool: "sandbox", action: "exec", sandbox_id: "box-a" }, result), "complete"), "ok");
assert.equal(toolExecutionOutput(result), result.result);
assert.equal(
toolRowKind(row({ tool: "execute" }, { code: 0, stdout: "fake [exit 7]", isError: false }), "complete"),
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.ts
index fd6175625..22d5a5b2a 100644
--- a/src/harness/agent-tools.ts
+++ b/src/harness/agent-tools.ts
@@ -710,7 +710,7 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
],
details: r,
},
- r.code !== 0 || r.timedOut,
+ r.timedOut,
undefined,
false,
undefined,
@@ -729,6 +729,12 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):
true,
);
}
+ await log("tool_result", {
+ ...sandboxLog({ tool: "execute", ...scopeNote }),
+ callId,
+ isError: true,
+ result: "Command execution failed.",
+ });
throw e;
}
};
diff --git a/test/agent-tools.test.ts b/test/agent-tools.test.ts
index ee418ab9c..47893ea2e 100644
--- a/test/agent-tools.test.ts
+++ b/test/agent-tools.test.ts
@@ -2968,7 +2968,7 @@ test("sandbox strict approvals remain action-scoped across resource activation",
});
for (const outcome of ["unscreened", "quarantine"] as const)
- test(`sandbox command failure preserves safe transcript metadata when output is ${outcome}`, async () => {
+ test(`sandbox nonzero exit preserves safe transcript metadata when output is ${outcome}`, async () => {
const entries: Emitted[] = [];
const tool = createAgentTools(
{
@@ -2988,7 +2988,7 @@ for (const outcome of ["unscreened", "quarantine"] as const)
const input = entries.find((e) => e.type === "tool_call")!.payload;
const output = entries.find((e) => e.type === "tool_result")!.payload;
assert.equal(input.sandbox_id, "box-a");
- assert.equal(output.isError, true);
+ assert.equal(output.isError, outcome === "quarantine");
assert.equal(output.stdout, undefined);
assert.equal(output.stderr, undefined);
if (outcome === "unscreened") {
@@ -3794,3 +3794,89 @@ for (const cancel of [false, true]) {
test("context recovery is not available in read-only mode", () => {
assert.ok(!createAgentTools({ current: fakeToolContext() }, { readOnly: true }).some((t) => t.name === "context"));
});
+
+test("command exit codes are data; timeouts and thrown tool errors are failures", async () => {
+ for (const sandboxResources of [false, true]) {
+ for (const outcome of ["zero", "nonzero", "timeout", "provider", "denied"] as const) {
+ const entries: Emitted[] = [];
+ const tool = createAgentTools(
+ {
+ current: {
+ ...fakeToolContext(),
+ execute: async () => {
+ if (outcome === "provider") throw new Error("sandbox provider unavailable");
+ if (outcome === "denied") throw new CommandDenied("[ -d dir ]", "policy denied");
+ return { stdout: "", stderr: "", code: outcome === "zero" ? 0 : 1, timedOut: outcome === "timeout" };
+ },
+ },
+ scopeLabel: "personal:U1",
+ emit: (entry) => {
+ entries.push(entry as Emitted);
+ },
+ },
+ { sandboxResources },
+ ).find((t) => t.name === (sandboxResources ? "sandbox" : "execute"))!;
+ const run = () =>
+ call(tool, {
+ ...(sandboxResources ? { action: "exec" } : {}),
+ command: "[ -d dir ]",
+ purpose: "Check directory",
+ });
+ if (outcome === "provider") {
+ await assert.rejects(run, /sandbox provider unavailable/);
+ const result = entries.find((e) => e.type === "tool_result")!.payload;
+ assert.equal(result.isError, true);
+ assert.equal(result.result, "Command execution failed.");
+ assert.equal(result.code, undefined);
+ continue;
+ }
+ const returned = await run();
+ const result = entries.find((e) => e.type === "tool_result")!.payload;
+ assert.equal(result.isError, !["zero", "nonzero"].includes(outcome), outcome);
+ if (outcome === "nonzero") {
+ assert.equal(result.code, 1);
+ assert.match(textOut(returned), /\[exit 1\]/);
+ }
+ }
+ }
+});
+
+test("thrown execution errors leave pending child messages for the next delivered result", async () => {
+ for (const sandboxResources of [false, true]) {
+ let pending = true;
+ const context = fakeToolContext();
+ context.execute = async () => {
+ throw new Error("provider unavailable");
+ };
+ context.sessionSyscalls = {
+ open: async () => ({ ok: false, message: "unused" }),
+ write: async () => ({ ok: false, message: "unused" }),
+ read: async () => ({ ok: false, message: "unused" }),
+ receive: async () =>
+ pending
+ ? [
+ {
+ id: "mail",
+ senderId: "child",
+ recipientId: "parent",
+ actor: { id: "U1", type: "internal" as const },
+ audience: [],
+ text: "Important child finding",
+ createdAt: 1,
+ },
+ ]
+ : [],
+ acknowledge: async () => {
+ pending = false;
+ },
+ };
+ const tools = createAgentTools({ current: context }, { sandboxResources });
+ const tool = tools.find((t) => t.name === (sandboxResources ? "sandbox" : "execute"))!;
+ const input = { ...(sandboxResources ? { action: "exec" } : {}), command: "true", purpose: "Check provider" };
+ await assert.rejects(() => call(tool, input), /provider unavailable/);
+ assert.equal(pending, true);
+ context.execute = async () => ({ stdout: "ok", stderr: "", code: 0, timedOut: false });
+ assert.match(JSON.stringify(await call(tool, input)), /Important child finding/);
+ assert.equal(pending, false);
+ }
+});