Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Verify persisted runtime handoff evidence in release qualification (#1095)

Regan Bellb3f119560592
diff --git a/test/live-slack-runtime-handoff.test.ts b/test/live-slack-runtime-handoff.test.ts
new file mode 100644
index 000000000..28241343f
--- /dev/null
+++ b/test/live-slack-runtime-handoff.test.ts
@@ -0,0 +1,58 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { assertRuntimeHandoff } from "./live-slack/runtime-handoff.ts";
+
+const call = (callId: string, action: string) => ({ type: "tool_call", payload: { tool: "runtime", callId, action } });
+const result = (callId: string, value: unknown) => ({
+ type: "tool_result",
+ payload: { tool: "runtime", callId, isError: false, result: JSON.stringify(value) },
+});
+const choice = { harnessId: "pi", modelId: "another-model" };
+const handoff = result("set-1", { ok: true, handoff: { lifetime: "task", choice } });
+const inspection = result("get-1", { ok: true, active: choice });
+const evidence = () => [call("set-1", "set"), handoff, call("get-1", "get"), inspection];
+
+test("runtime handoff qualification reads persisted JSON tool results", () => {
+ assert.deepEqual(assertRuntimeHandoff(evidence()), choice);
+});
+
+test("runtime handoff qualification rejects prose, uncorrelated results, errors, and wrong actions", () => {
+ const invalid = [
+ [{ type: "assistant", payload: { text: JSON.stringify(handoff) } }],
+ [handoff, call("get-1", "get"), inspection],
+ [call("set-1", "get"), handoff, call("get-1", "get"), inspection],
+ [
+ call("set-1", "set"),
+ { ...handoff, payload: { ...handoff.payload, isError: true } },
+ call("get-1", "get"),
+ inspection,
+ ],
+ [
+ call("set-1", "set"),
+ result("set-1", { ok: false, handoff: { lifetime: "task", choice } }),
+ call("get-1", "get"),
+ inspection,
+ ],
+ ];
+ for (const entries of invalid) assert.throws(() => assertRuntimeHandoff(entries), /handoff recorded/);
+});
+
+test("runtime inspection must follow the handoff and confirm the selected model and harness", () => {
+ assert.throws(() => assertRuntimeHandoff(evidence().slice(0, 2)), /inspection after handoff/);
+ assert.throws(
+ () => assertRuntimeHandoff([call("get-1", "get"), inspection, call("set-1", "set"), handoff]),
+ /inspection after handoff/,
+ );
+ for (const active of [
+ { ...choice, modelId: "old-model" },
+ { ...choice, harnessId: "other" },
+ ])
+ assert.throws(() =>
+ assertRuntimeHandoff([
+ call("set-1", "set"),
+ handoff,
+ call("get-1", "get"),
+ result("get-1", { ok: true, active }),
+ ]),
+ );
+});
diff --git a/test/live-slack/runtime-handoff.ts b/test/live-slack/runtime-handoff.ts
new file mode 100644
index 000000000..4e5d02dd9
--- /dev/null
+++ b/test/live-slack/runtime-handoff.ts
@@ -0,0 +1,40 @@
+import assert from "node:assert/strict";
+import { isObj } from "../../src/util/objects.ts";
+
+export function assertRuntimeHandoff(entries: unknown[]): { harnessId: string; modelId: string } {
+ const calls = new Map<string, { action: string; index: number }>();
+ const results: Array<{ index: number; action: string; value: Record<string, unknown> }> = [];
+ for (const [index, entry] of entries.entries()) {
+ if (!isObj(entry) || !isObj(entry.payload)) continue;
+ const payload = entry.payload;
+ if (payload.tool !== "runtime" || typeof payload.callId !== "string") continue;
+ if (entry.type === "tool_call" && typeof payload.action === "string") {
+ calls.set(payload.callId, { action: payload.action, index });
+ continue;
+ }
+ const call = calls.get(payload.callId);
+ if (entry.type !== "tool_result" || payload.isError !== false || !call || call.index >= index) continue;
+ if (typeof payload.result !== "string") continue;
+ let value: unknown;
+ try {
+ value = JSON.parse(payload.result);
+ } catch {
+ continue;
+ }
+ if (isObj(value) && value.ok === true) results.push({ index, action: call.action, value });
+ }
+ const handoff = results.find((result) => result.action === "set" && isObj(result.value.handoff));
+ assert.ok(handoff && isObj(handoff.value.handoff), "no successful correlated runtime handoff recorded");
+ const { lifetime, choice } = handoff.value.handoff;
+ assert.equal(lifetime, "task");
+ assert.ok(isObj(choice));
+ assert.equal(choice.harnessId, "pi");
+ assert.ok(typeof choice.modelId === "string" && choice.modelId.length > 0);
+ const verification = results.find(
+ (result) => result.index > handoff.index && result.action === "get" && isObj(result.value.active),
+ );
+ assert.ok(verification && isObj(verification.value.active), "no successful runtime inspection after handoff");
+ assert.equal(verification.value.active.modelId, choice.modelId);
+ assert.equal(verification.value.active.harnessId, choice.harnessId);
+ return { harnessId: "pi", modelId: choice.modelId };
+}
diff --git a/test/live-slack/scenarios.ts b/test/live-slack/scenarios.ts
index 0fb4badc6..a84842d3a 100644
--- a/test/live-slack/scenarios.ts
+++ b/test/live-slack/scenarios.ts
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
import path from "node:path";
import { assert, isLiveStatusText, type Scenario } from "./harness.ts";
import { sleep } from "./slack.ts";
+import { assertRuntimeHandoff } from "./runtime-handoff.ts";
import { multiUserScenarios } from "./scenarios-multiuser.ts";
import { twinScenarios } from "./scenarios-twin.ts";
@@ -28,34 +29,13 @@ export const scenarios: Scenario[] = [
const ch = await ctx.freshChannel();
const marker = ctx.marker();
const root = await ch.mention(
- `Can you switch your model to a different available Anthropic model for this request and then calculate 17 × 23? Keep using pi with automatic reasoning effort and fast mode off. Once you have switched, verify which model you are actually running on. Include ${marker}, that model and the answer in your final reply.`,
+ `Use runtime action=get to inspect the available models, then runtime action=set to switch to a different available Anthropic model for this request. Keep using pi with automatic reasoning effort and fast mode off. After the handoff, call runtime action=get again to verify which model you are actually running on, then calculate 17 × 23. Include ${marker}, that model and the answer in your final reply.`,
);
const reply = await ch.waitForBotReply(root, { match: new RegExp(marker), timeoutMs: 4 * 60_000 });
assert.match(reply.text ?? "", /\b391\b/);
const session = await ctx.core.findSessionByThread(ch.id, root);
assert.ok(session, "no core session found for runtime handoff");
- const handoffEntry = session.entries.find(
- (entry: any) => entry.type === "tool_result" && entry.payload?.runtimeHandoff,
- ) as
- | { payload: { runtimeHandoff: { lifetime: string; choice: { harnessId: string; modelId: string } } } }
- | undefined;
- assert.ok(handoffEntry, "no durable runtime handoff recorded");
- const { lifetime, choice } = handoffEntry.payload.runtimeHandoff;
- assert.equal(lifetime, "task");
- assert.equal(choice.harnessId, "pi");
- const handoffIndex = session.entries.indexOf(handoffEntry);
- const verification = session.entries
- .slice(handoffIndex + 1)
- .find(
- (entry: any) =>
- entry.type === "tool_result" &&
- entry.payload?.tool === "runtime" &&
- entry.payload?.action === "get" &&
- entry.payload?.ok === true,
- ) as { payload: { result: string } } | undefined;
- assert.ok(verification, "no successful runtime inspection after handoff");
- const inspected = JSON.parse(verification.payload.result) as { active: { modelId: string } };
- assert.equal(inspected.active.modelId, choice.modelId);
+ const choice = assertRuntimeHandoff(session.entries);
type ModelCall = { step: number; model: string; createdAt: number; usage: { output: number } | null };
let modelCalls: ModelCall[];
const deadline = Date.now() + 30_000;