Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Verify projected sandbox results in cloud qualification (#1099)

Regan Bell32b38cec6eff
diff --git a/test/live-slack-sandbox-providers.test.ts b/test/live-slack-sandbox-providers.test.ts
index 9bb258702..b332144d4 100644
--- a/test/live-slack-sandbox-providers.test.ts
+++ b/test/live-slack-sandbox-providers.test.ts
@@ -38,6 +38,28 @@ test("provider execution requires correlated success on the exact sandbox", () =
assert.throws(() => assertSandboxExecution([{ type: "assistant", payload: { text: "nonce" } }], "box", "nonce\n"));
});
+test("projected sandbox results require exact successful execution evidence", () => {
+ const entries = evidence() as Array<{ type: string; payload: Record<string, unknown> }>;
+ entries[1]!.payload = { tool: "sandbox", callId: "call", isError: false, result: "nonce\n\n[exit 0]" };
+ assert.doesNotThrow(() => assertSandboxExecution(entries, "box", "nonce\n"));
+ for (const patch of [
+ { result: "nonce\n" },
+ { result: "nonce\n\n[exit 1]" },
+ { result: "nonce\n\n[exit 0 timed-out]" },
+ { result: "wrong\n\n[exit 0]" },
+ { result: "nonce\n\n[stderr]\nfailed\n[exit 0]" },
+ { isError: true },
+ { callId: "unrelated" },
+ { code: 1 },
+ { timedOut: true },
+ { action: "start_process" },
+ ]) {
+ const changed = [entries[0], { ...entries[1], payload: { ...entries[1]!.payload, ...patch } }];
+ assert.throws(() => assertSandboxExecution(changed, "box", "nonce\n"));
+ }
+ assert.throws(() => assertSandboxExecution(entries.toReversed(), "box", "nonce\n"));
+});
+
test("every provider has its own parallel execution scenario", () => {
assert.deepEqual(sandboxProviders.toSorted(), [
"agent37",
diff --git a/test/live-slack/scenarios-sandbox-providers.ts b/test/live-slack/scenarios-sandbox-providers.ts
index 8d632034c..20d3b5b08 100644
--- a/test/live-slack/scenarios-sandbox-providers.ts
+++ b/test/live-slack/scenarios-sandbox-providers.ts
@@ -18,33 +18,27 @@ const providerCoverage: Record<SandboxBackendName, true> = {
export const sandboxProviders = Object.keys(providerCoverage) as SandboxBackendName[];
export function assertSandboxExecution(entries: readonly unknown[], sandboxId: string, stdout: string): void {
- const calls = new Set(
- entries.flatMap((entry) => {
- if (!isObj(entry) || entry.type !== "tool_call" || !isObj(entry.payload)) return [];
- const p = entry.payload;
- return p.tool === "sandbox" && p.action === "exec" && p.sandbox_id === sandboxId && typeof p.callId === "string"
- ? [p.callId]
- : [];
- }),
- );
+ const calls = new Set<string>();
+ let executed = false;
+ for (const entry of entries) {
+ if (!isObj(entry) || !isObj(entry.payload)) continue;
+ const p = entry.payload;
+ if (p.tool !== "sandbox" || typeof p.callId !== "string") continue;
+ if (entry.type === "tool_call" && p.action === "exec" && p.sandbox_id === sandboxId) {
+ calls.add(p.callId);
+ continue;
+ }
+ if (entry.type !== "tool_result" || !calls.has(p.callId) || p.isError !== false) continue;
+ const structured = "code" in p || "timedOut" in p || "stdout" in p || "action" in p;
+ if (
+ structured
+ ? p.action === "exec" && p.code === 0 && p.timedOut === false && p.stdout === stdout
+ : p.result === `${stdout}\n[exit 0]`
+ )
+ executed = true;
+ }
assert.ok(calls.size, `no sandbox exec call targeted ${sandboxId}`);
- assert.ok(
- entries.some((entry) => {
- if (!isObj(entry) || entry.type !== "tool_result" || !isObj(entry.payload)) return false;
- const p = entry.payload;
- return (
- p.tool === "sandbox" &&
- p.action === "exec" &&
- typeof p.callId === "string" &&
- calls.has(p.callId) &&
- p.isError === false &&
- p.code === 0 &&
- p.timedOut === false &&
- p.stdout === stdout
- );
- }),
- `no successful sandbox exec result with exact stdout on ${sandboxId}`,
- );
+ assert.ok(executed, `no successful sandbox exec result with exact stdout on ${sandboxId}`);
}
export const sandboxProviderScenarios: Scenario[] = sandboxProviders.map((backend) => ({