Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Check sandbox execution against the stored result (#2345)

Regan Bell8c014f358d13
diff --git a/test/live-slack-catalog.test.ts b/test/live-slack-catalog.test.ts
index 4356e3deb..a8f20070c 100644
--- a/test/live-slack-catalog.test.ts
+++ b/test/live-slack-catalog.test.ts
@@ -124,7 +124,14 @@ for (const [name, tool] of [
entries: [
{
type: "tool_result",
- payload: { tool, action: "exec", code: 0, isError: false, timedOut: false, stdout: `${output}\n` },
+ payload: {
+ tool,
+ action: "exec",
+ code: 0,
+ isError: false,
+ timedOut: false,
+ result: `${output}\n\n[exit 0]`,
+ },
},
],
}),
diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.ts
index 5a56c7a20..0b5fd52df 100644
--- a/test/live-slack-sandbox-providers.test.ts
+++ b/test/live-slack-sandbox-providers.test.ts
@@ -13,7 +13,14 @@ function evidence(sandboxId = "box", stdout = "nonce\n"): unknown[] {
{ type: "tool_call", payload: { tool: "execute", sandbox_id: sandboxId, callId: "call" } },
{
type: "tool_result",
- payload: { tool: "execute", callId: "call", isError: false, code: 0, timedOut: false, stdout },
+ payload: {
+ tool: "execute",
+ callId: "call",
+ isError: false,
+ code: 0,
+ timedOut: false,
+ result: `${stdout}\n[exit 0]`,
+ },
},
];
}
@@ -27,8 +34,9 @@ test("provider execution requires correlated success on the exact sandbox", () =
{ isError: true },
{ code: 1 },
{ timedOut: true },
- { stdout: "wrong\n" },
- { stdout: undefined, result: "nonce\n" },
+ { result: "wrong\n\n[exit 0]" },
+ { result: "nonce\n" },
+ { result: undefined, stdout: "nonce\n" },
]) {
const entries = evidence() as Array<{ type: string; payload: Record<string, unknown> }>;
Object.assign(entries[1]!.payload, patch);
diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.ts
index a4ec2f0f3..415b050c1 100644
--- a/test/live-slack/scenarios-sandbox-providers.ts
+++ b/test/live-slack/scenarios-sandbox-providers.ts
@@ -19,6 +19,13 @@ const providerCoverage: Record<SandboxBackendName, true> = {
export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];
+export function succeededWithStdout(p: Record<string, unknown>, stdout: string): boolean {
+ const structured = "code" in p || "timedOut" in p || "action" in p;
+ return (
+ p.isError === false && p.result === `${stdout}\n[exit 0]` && (!structured || (p.code === 0 && p.timedOut === false))
+ );
+}
+
export function assertSandboxExecution(entries: readonly unknown[], sandboxId: string, stdout: string): void {
const calls = new Set<string>();
let executed = false;
@@ -30,14 +37,7 @@ export function assertSandboxExecution(entries: readonly unknown[], sandboxId: s
calls.add(p.callId);
continue;
}
- if (entry.type !== "tool_result" || !calls.has(p.callId) || p.isError !== false) continue;
- const structured = "code" in p || "timedOut" in p || "stdout" in p || "action" in p;
- if (
- structured
- ? p.background !== true && p.code === 0 && p.timedOut === false && p.stdout === stdout
- : p.result === `${stdout}\n[exit 0]`
- )
- executed = true;
+ if (entry.type === "tool_result" && calls.has(p.callId) && succeededWithStdout(p, stdout)) executed = true;
}
assert.ok(calls.size, `no execute call targeted ${sandboxId}`);
assert.ok(executed, `no successful execute result with exact stdout on ${sandboxId}`);
diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.ts
index c2c4aae03..3ff0bdb81 100644
--- a/test/live-slack/scenarios.ts
+++ b/test/live-slack/scenarios.ts
@@ -3,6 +3,7 @@ import path from "node:path";
import { assert, type Scenario } from "./harness.ts";
import { sleep, type SlackMessage } from "./slack.ts";
import { assertRuntimeHandoff } from "./runtime-handoff.ts";
+import { succeededWithStdout } from "./scenarios-sandbox-providers.ts";
import { multiUserScenarios } from "./scenarios-multiuser.ts";
import { twinScenarios } from "./scenarios-twin.ts";
import { deployAccessScenarios } from "./scenarios-deploy-access.ts";
@@ -177,10 +178,7 @@ export const scenarios: Scenario[] = [
(e: any) =>
e.type === "tool_result" &&
(e.payload?.tool === "execute" || (e.payload?.tool === "sandbox" && e.payload.action === "exec")) &&
- e.payload.isError === false &&
- e.payload.code === 0 &&
- e.payload.timedOut === false &&
- e.payload.stdout === `${marker}\n`,
+ succeededWithStdout(e.payload, `${marker}\n`),
);
assert.ok(ran, "core transcript has no successful command execution — cannot evaluate performance");