Multiplayer agent harness for work.
Verify projected sandbox results in cloud qualification (#1099)
Regan Bell32b38cec6eff
diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.tsindex 9bb258702..b332144d4 100644--- a/test/live-slack-sandbox-providers.test.ts+++ b/test/live-slack-sandbox-providers.test.ts@@ -38,6 +38,28 @@ test("provider execution requires correlated success on the exact sandbox", () =assert.throws(() => assertSandboxExecution([{ type: "assistant", payload: { text: "nonce" } }], "box", "nonce\n"));});+test("projected sandbox results require exact successful execution evidence", () => {+ const entries = evidence() as Array<{ type: string; payload: Record<string, unknown> }>;+ entries[1]!.payload = { tool: "sandbox", callId: "call", isError: false, result: "nonce\n\n[exit 0]" };+ assert.doesNotThrow(() => assertSandboxExecution(entries, "box", "nonce\n"));+ for (const patch of [+ { result: "nonce\n" },+ { result: "nonce\n\n[exit 1]" },+ { result: "nonce\n\n[exit 0 timed-out]" },+ { result: "wrong\n\n[exit 0]" },+ { result: "nonce\n\n[stderr]\nfailed\n[exit 0]" },+ { isError: true },+ { callId: "unrelated" },+ { code: 1 },+ { timedOut: true },+ { action: "start_process" },+ ]) {+ const changed = [entries[0], { ...entries[1], payload: { ...entries[1]!.payload, ...patch } }];+ assert.throws(() => assertSandboxExecution(changed, "box", "nonce\n"));+ }+ assert.throws(() => assertSandboxExecution(entries.toReversed(), "box", "nonce\n"));+});+test("every provider has its own parallel execution scenario", () => {assert.deepEqual(sandboxProviders.toSorted(), ["agent37",diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.tsindex 8d632034c..20d3b5b08 100644--- a/test/live-slack/scenarios-sandbox-providers.ts+++ b/test/live-slack/scenarios-sandbox-providers.ts@@ -18,33 +18,27 @@ const providerCoverage: Record<SandboxBackendName, true> = {export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];export function assertSandboxExecution(entries: readonly unknown[], sandboxId: string, stdout: string): void {- const calls = new Set(- entries.flatMap((entry) => {- if (!isObj(entry) || entry.type !== "tool_call" || !isObj(entry.payload)) return [];- const p = entry.payload;- return p.tool === "sandbox" && p.action === "exec" && p.sandbox_id === sandboxId && typeof p.callId === "string"- ? [p.callId]- : [];- }),- );+ const calls = new Set<string>();+ let executed = false;+ for (const entry of entries) {+ if (!isObj(entry) || !isObj(entry.payload)) continue;+ const p = entry.payload;+ if (p.tool !== "sandbox" || typeof p.callId !== "string") continue;+ if (entry.type === "tool_call" && p.action === "exec" && p.sandbox_id === sandboxId) {+ calls.add(p.callId);+ continue;+ }+ if (entry.type !== "tool_result" || !calls.has(p.callId) || p.isError !== false) continue;+ const structured = "code" in p || "timedOut" in p || "stdout" in p || "action" in p;+ if (+ structured+ ? p.action === "exec" && p.code === 0 && p.timedOut === false && p.stdout === stdout+ : p.result === `${stdout}\n[exit 0]`+ )+ executed = true;+ }assert.ok(calls.size, `no sandbox exec call targeted ${sandboxId}`);- assert.ok(- entries.some((entry) => {- if (!isObj(entry) || entry.type !== "tool_result" || !isObj(entry.payload)) return false;- const p = entry.payload;- return (- p.tool === "sandbox" &&- p.action === "exec" &&- typeof p.callId === "string" &&- calls.has(p.callId) &&- p.isError === false &&- p.code === 0 &&- p.timedOut === false &&- p.stdout === stdout- );- }),- `no successful sandbox exec result with exact stdout on ${sandboxId}`,- );+ assert.ok(executed, `no successful sandbox exec result with exact stdout on ${sandboxId}`);}export const sandboxProviderScenarios: Scenario[] = sandboxProviders.map((backend) => ({