Multiplayer agent harness for work.
Check sandbox execution against the stored result (#2345)
Regan Bell8c014f358d13
diff --git a/test/live-slack-catalog.test.ts b/test/live-slack-catalog.test.tsindex 4356e3deb..a8f20070c 100644--- a/test/live-slack-catalog.test.ts+++ b/test/live-slack-catalog.test.ts@@ -124,7 +124,14 @@ for (const [name, tool] of [entries: [{type: "tool_result",- payload: { tool, action: "exec", code: 0, isError: false, timedOut: false, stdout: `${output}\n` },+ payload: {+ tool,+ action: "exec",+ code: 0,+ isError: false,+ timedOut: false,+ result: `${output}\n\n[exit 0]`,+ },},],}),diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.tsindex 5a56c7a20..0b5fd52df 100644--- a/test/live-slack-sandbox-providers.test.ts+++ b/test/live-slack-sandbox-providers.test.ts@@ -13,7 +13,14 @@ function evidence(sandboxId = "box", stdout = "nonce\n"): unknown[] {{ type: "tool_call", payload: { tool: "execute", sandbox_id: sandboxId, callId: "call" } },{type: "tool_result",- payload: { tool: "execute", callId: "call", isError: false, code: 0, timedOut: false, stdout },+ payload: {+ tool: "execute",+ callId: "call",+ isError: false,+ code: 0,+ timedOut: false,+ result: `${stdout}\n[exit 0]`,+ },},];}@@ -27,8 +34,9 @@ test("provider execution requires correlated success on the exact sandbox", () ={ isError: true },{ code: 1 },{ timedOut: true },- { stdout: "wrong\n" },- { stdout: undefined, result: "nonce\n" },+ { result: "wrong\n\n[exit 0]" },+ { result: "nonce\n" },+ { result: undefined, stdout: "nonce\n" },]) {const entries = evidence() as Array<{ type: string; payload: Record<string, unknown> }>;Object.assign(entries[1]!.payload, patch);diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.tsindex a4ec2f0f3..415b050c1 100644--- a/test/live-slack/scenarios-sandbox-providers.ts+++ b/test/live-slack/scenarios-sandbox-providers.ts@@ -19,6 +19,13 @@ const providerCoverage: Record<SandboxBackendName, true> = {export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];+export function succeededWithStdout(p: Record<string, unknown>, stdout: string): boolean {+ const structured = "code" in p || "timedOut" in p || "action" in p;+ return (+ p.isError === false && p.result === `${stdout}\n[exit 0]` && (!structured || (p.code === 0 && p.timedOut === false))+ );+}+export function assertSandboxExecution(entries: readonly unknown[], sandboxId: string, stdout: string): void {const calls = new Set<string>();let executed = false;@@ -30,14 +37,7 @@ export function assertSandboxExecution(entries: readonly unknown[], sandboxId: scalls.add(p.callId);continue;}- if (entry.type !== "tool_result" || !calls.has(p.callId) || p.isError !== false) continue;- const structured = "code" in p || "timedOut" in p || "stdout" in p || "action" in p;- if (- structured- ? p.background !== true && p.code === 0 && p.timedOut === false && p.stdout === stdout- : p.result === `${stdout}\n[exit 0]`- )- executed = true;+ if (entry.type === "tool_result" && calls.has(p.callId) && succeededWithStdout(p, stdout)) executed = true;}assert.ok(calls.size, `no execute call targeted ${sandboxId}`);assert.ok(executed, `no successful execute result with exact stdout on ${sandboxId}`);diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.tsindex c2c4aae03..3ff0bdb81 100644--- a/test/live-slack/scenarios.ts+++ b/test/live-slack/scenarios.ts@@ -3,6 +3,7 @@ import path from "node:path";import { assert, type Scenario } from "./harness.ts";import { sleep, type SlackMessage } from "./slack.ts";import { assertRuntimeHandoff } from "./runtime-handoff.ts";+import { succeededWithStdout } from "./scenarios-sandbox-providers.ts";import { multiUserScenarios } from "./scenarios-multiuser.ts";import { twinScenarios } from "./scenarios-twin.ts";import { deployAccessScenarios } from "./scenarios-deploy-access.ts";@@ -177,10 +178,7 @@ export const scenarios: Scenario[] = [(e: any) =>e.type === "tool_result" &&(e.payload?.tool === "execute" || (e.payload?.tool === "sandbox" && e.payload.action === "exec")) &&- e.payload.isError === false &&- e.payload.code === 0 &&- e.payload.timedOut === false &&- e.payload.stdout === `${marker}\n`,+ succeededWithStdout(e.payload, `${marker}\n`),);assert.ok(ran, "core transcript has no successful command execution — cannot evaluate performance");