Multiplayer agent harness for work.
Verify persisted runtime handoff evidence in release qualification (#1095)
Regan Bellb3f119560592
diff --git a/test/live-slack-runtime-handoff.test.ts b/test/live-slack-runtime-handoff.test.tsnew file mode 100644index 000000000..28241343f--- /dev/null+++ b/test/live-slack-runtime-handoff.test.ts@@ -0,0 +1,58 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { assertRuntimeHandoff } from "./live-slack/runtime-handoff.ts";++const call = (callId: string, action: string) => ({ type: "tool_call", payload: { tool: "runtime", callId, action } });+const result = (callId: string, value: unknown) => ({+ type: "tool_result",+ payload: { tool: "runtime", callId, isError: false, result: JSON.stringify(value) },+});+const choice = { harnessId: "pi", modelId: "another-model" };+const handoff = result("set-1", { ok: true, handoff: { lifetime: "task", choice } });+const inspection = result("get-1", { ok: true, active: choice });+const evidence = () => [call("set-1", "set"), handoff, call("get-1", "get"), inspection];++test("runtime handoff qualification reads persisted JSON tool results", () => {+ assert.deepEqual(assertRuntimeHandoff(evidence()), choice);+});++test("runtime handoff qualification rejects prose, uncorrelated results, errors, and wrong actions", () => {+ const invalid = [+ [{ type: "assistant", payload: { text: JSON.stringify(handoff) } }],+ [handoff, call("get-1", "get"), inspection],+ [call("set-1", "get"), handoff, call("get-1", "get"), inspection],+ [+ call("set-1", "set"),+ { ...handoff, payload: { ...handoff.payload, isError: true } },+ call("get-1", "get"),+ inspection,+ ],+ [+ call("set-1", "set"),+ result("set-1", { ok: false, handoff: { lifetime: "task", choice } }),+ call("get-1", "get"),+ inspection,+ ],+ ];+ for (const entries of invalid) assert.throws(() => assertRuntimeHandoff(entries), /handoff recorded/);+});++test("runtime inspection must follow the handoff and confirm the selected model and harness", () => {+ assert.throws(() => assertRuntimeHandoff(evidence().slice(0, 2)), /inspection after handoff/);+ assert.throws(+ () => assertRuntimeHandoff([call("get-1", "get"), inspection, call("set-1", "set"), handoff]),+ /inspection after handoff/,+ );+ for (const active of [+ { ...choice, modelId: "old-model" },+ { ...choice, harnessId: "other" },+ ])+ assert.throws(() =>+ assertRuntimeHandoff([+ call("set-1", "set"),+ handoff,+ call("get-1", "get"),+ result("get-1", { ok: true, active }),+ ]),+ );+});diff --git a/test/live-slack/runtime-handoff.ts b/test/live-slack/runtime-handoff.tsnew file mode 100644index 000000000..4e5d02dd9--- /dev/null+++ b/test/live-slack/runtime-handoff.ts@@ -0,0 +1,40 @@+import assert from "node:assert/strict";+import { isObj } from "../../src/util/objects.ts";++export function assertRuntimeHandoff(entries: unknown[]): { harnessId: string; modelId: string } {+ const calls = new Map<string, { action: string; index: number }>();+ const results: Array<{ index: number; action: string; value: Record<string, unknown> }> = [];+ for (const [index, entry] of entries.entries()) {+ if (!isObj(entry) || !isObj(entry.payload)) continue;+ const payload = entry.payload;+ if (payload.tool !== "runtime" || typeof payload.callId !== "string") continue;+ if (entry.type === "tool_call" && typeof payload.action === "string") {+ calls.set(payload.callId, { action: payload.action, index });+ continue;+ }+ const call = calls.get(payload.callId);+ if (entry.type !== "tool_result" || payload.isError !== false || !call || call.index >= index) continue;+ if (typeof payload.result !== "string") continue;+ let value: unknown;+ try {+ value = JSON.parse(payload.result);+ } catch {+ continue;+ }+ if (isObj(value) && value.ok === true) results.push({ index, action: call.action, value });+ }+ const handoff = results.find((result) => result.action === "set" && isObj(result.value.handoff));+ assert.ok(handoff && isObj(handoff.value.handoff), "no successful correlated runtime handoff recorded");+ const { lifetime, choice } = handoff.value.handoff;+ assert.equal(lifetime, "task");+ assert.ok(isObj(choice));+ assert.equal(choice.harnessId, "pi");+ assert.ok(typeof choice.modelId === "string" && choice.modelId.length > 0);+ const verification = results.find(+ (result) => result.index > handoff.index && result.action === "get" && isObj(result.value.active),+ );+ assert.ok(verification && isObj(verification.value.active), "no successful runtime inspection after handoff");+ assert.equal(verification.value.active.modelId, choice.modelId);+ assert.equal(verification.value.active.harnessId, choice.harnessId);+ return { harnessId: "pi", modelId: choice.modelId };+}diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.tsindex 0fb4badc6..a84842d3a 100644--- a/test/live-slack/scenarios.ts+++ b/test/live-slack/scenarios.ts@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";import path from "node:path";import { assert, isLiveStatusText, type Scenario } from "./harness.ts";import { sleep } from "./slack.ts";+import { assertRuntimeHandoff } from "./runtime-handoff.ts";import { multiUserScenarios } from "./scenarios-multiuser.ts";import { twinScenarios } from "./scenarios-twin.ts";@@ -28,34 +29,13 @@ export const scenarios: Scenario[] = [const ch = await ctx.freshChannel();const marker = ctx.marker();const root = await ch.mention(- `Can you switch your model to a different available Anthropic model for this request and then calculate 17 × 23? Keep using pi with automatic reasoning effort and fast mode off. Once you have switched, verify which model you are actually running on. Include ${marker}, that model and the answer in your final reply.`,+ `Use runtime action=get to inspect the available models, then runtime action=set to switch to a different available Anthropic model for this request. Keep using pi with automatic reasoning effort and fast mode off. After the handoff, call runtime action=get again to verify which model you are actually running on, then calculate 17 × 23. Include ${marker}, that model and the answer in your final reply.`,);const reply = await ch.waitForBotReply(root, { match: new RegExp(marker), timeoutMs: 4 * 60_000 });assert.match(reply.text ?? "", /\b391\b/);const session = await ctx.core.findSessionByThread(ch.id, root);assert.ok(session, "no core session found for runtime handoff");- const handoffEntry = session.entries.find(- (entry: any) => entry.type === "tool_result" && entry.payload?.runtimeHandoff,- ) as- | { payload: { runtimeHandoff: { lifetime: string; choice: { harnessId: string; modelId: string } } } }- | undefined;- assert.ok(handoffEntry, "no durable runtime handoff recorded");- const { lifetime, choice } = handoffEntry.payload.runtimeHandoff;- assert.equal(lifetime, "task");- assert.equal(choice.harnessId, "pi");- const handoffIndex = session.entries.indexOf(handoffEntry);- const verification = session.entries- .slice(handoffIndex + 1)- .find(- (entry: any) =>- entry.type === "tool_result" &&- entry.payload?.tool === "runtime" &&- entry.payload?.action === "get" &&- entry.payload?.ok === true,- ) as { payload: { result: string } } | undefined;- assert.ok(verification, "no successful runtime inspection after handoff");- const inspected = JSON.parse(verification.payload.result) as { active: { modelId: string } };- assert.equal(inspected.active.modelId, choice.modelId);+ const choice = assertRuntimeHandoff(session.entries);type ModelCall = { step: number; model: string; createdAt: number; usage: { output: number } | null };let modelCalls: ModelCall[];const deadline = Date.now() + 30_000;