Multiplayer agent harness for work.
Treat a goal governor that cannot decide as no decision (#2536)
Joshua France9ed193be61ff
diff --git a/src/harness/agent-tools.ts b/src/harness/agent-tools.tsindex 70c2c6230..51739f7b1 100644--- a/src/harness/agent-tools.ts+++ b/src/harness/agent-tools.ts@@ -40,6 +40,8 @@ import {GOAL_STEP_BACK_PROMPT,type GoalGovernor,type GoalRecord,+ type GovernorVerdict,+ GovernorUndecided,} from "./goal.ts";import {toolLabelOf,@@ -4089,23 +4091,26 @@ export function createAgentTools(ref: ToolContextRef, opts?: AgentToolsOptions):p.note ?? "",...(await goalEvidenceFiles(ref.current, p.files, ref.abortSignal, p.sandbox_id)),].join("\n");- const verdict = ref.governGoal- ? await ref- .governGoal({- objective: goal.objective,- trigger: "completion",- recentWork: ref.goalRecentWork?.() ?? "",- evidence,- ...(goal.governor ? { previous: goal.governor } : {}),- })- .catch((e: unknown) => ({- verdict: "continue" as const,- reasons: `the governor failed (${errMessage(e)}); request completion again`,- }))- : {- verdict: "continue" as const,- reasons: "no independent governor is available on this runtime; keep working",- };+ let verdict: GovernorVerdict;+ try {+ if (!ref.governGoal) throw new GovernorUndecided("no independent governor is available on this runtime");+ verdict = await ref.governGoal({+ objective: goal.objective,+ trigger: "completion",+ recentWork: ref.goalRecentWork?.() ?? "",+ evidence,+ ...(goal.governor ? { previous: goal.governor } : {}),+ });+ } catch (e) {+ return recordCoreAuthoredResult(+ callId,+ { tool: "goal", action: "update", error: "governor_undecided", goal },+ text(+ `The governor could not reach a decision (${errMessage(e)}). This is not a rejection and nothing was recorded against the goal; it stays active. Request completion again with the same evidence.`,+ ),+ true,+ );+ }if (verdict.verdict !== "complete") {const applied = applyGovernorVerdict(goal, verdict);await ref.saveGoal?.(goal);diff --git a/src/harness/goal.ts b/src/harness/goal.tsindex 98ff4b288..d8a1693d6 100644--- a/src/harness/goal.ts+++ b/src/harness/goal.ts@@ -32,6 +32,7 @@ import { Type, type Static } from "typebox";import type { LlmCallUsage } from "../sessions/session-store.ts";import { type GrindBudget, type GrindMeter, grindState } from "./grind.ts";import { withoutWire } from "./context-compaction.ts";+import { swallowAs } from "../util/errors.ts";type GoalStatus = "active" | "paused" | "complete" | "cancelled";@@ -318,7 +319,6 @@ export function goalWorkDigest(lines: ReadonlyArray<string>, maxChars = GOAL_WORreturn kept.reverse().join("\n");}-/** A fresh-context judge call. Fails closed to "continue". */export async function governGoal(judge: (system: string, prompt: string, signal?: AbortSignal) => Promise<string | undefined>,input: GoalGovernorInput,@@ -336,23 +336,29 @@ export async function governGoal(.filter(Boolean).join("\n");const reply = (await judge(GOAL_GOVERNOR_SYSTEM_PROMPT, prompt, signal)) ?? "";- let verdict: GovernorVerdict["verdict"] = "continue";- let reasons = "the governor's reply was not parseable";+ let parsed: { verdict?: unknown; reasons?: unknown } | undefined;try {- const parsed = JSON.parse(reply.slice(reply.indexOf("{"), reply.lastIndexOf("}") + 1)) as {- verdict?: unknown;- reasons?: unknown;- };- if (["continue", "complete", "step_back", "pause"].includes(parsed.verdict as string))- verdict = parsed.verdict as GovernorVerdict["verdict"];- reasons = typeof parsed.reasons === "string" && parsed.reasons.trim() ? parsed.reasons.trim() : "no reasons given";+ parsed = JSON.parse(reply.slice(reply.indexOf("{"), reply.lastIndexOf("}") + 1)) as typeof parsed;} catch {- /* fail closed below */+ parsed = undefined;}+ if (!parsed || !GOVERNOR_VERDICTS.includes(parsed.verdict as GovernorVerdict["verdict"]))+ throw new GovernorUndecided(+ reply.trim()+ ? `its reply was not a verdict: ${JSON.stringify(reply.trim().slice(0, 200))}`+ : "it returned no reply",+ );+ let verdict = parsed.verdict as GovernorVerdict["verdict"];+ const reasons =+ typeof parsed.reasons === "string" && parsed.reasons.trim() ? parsed.reasons.trim() : "no reasons given";if (verdict === "complete" && input.trigger !== "completion") verdict = "continue";return { verdict, reasons };}+const GOVERNOR_VERDICTS: ReadonlyArray<GovernorVerdict["verdict"]> = ["continue", "complete", "step_back", "pause"];++export class GovernorUndecided extends Error {}+/*** Record a verdict on the goal and return the one that took effect. A pause is* honored only right after a step back (otherwise it becomes the step back);@@ -525,10 +531,9 @@ export async function enforceGoal<T>(opts: {) {rounds = 0;lastCheck = now();- const verdict = await opts- .govern(opts.goal.governor)- .catch((e: unknown) => ({ verdict: "continue" as const, reasons: `the governor failed (${String(e)})` }));- applyGovernorVerdict(opts.goal, verdict.verdict === "complete" ? { ...verdict, verdict: "continue" } : verdict);+ const verdict = await opts.govern(opts.goal.governor).catch(swallowAs("goal: checkpoint governor", undefined));+ if (verdict)+ applyGovernorVerdict(opts.goal, verdict.verdict === "complete" ? { ...verdict, verdict: "continue" } : verdict);}rounds++;if (opts.goal.status === "paused") note = goalGovernorPausePrompt(opts.goal);diff --git a/test/goal-tools.test.ts b/test/goal-tools.test.tsindex aaffaa4fa..45a95cc97 100644--- a/test/goal-tools.test.ts+++ b/test/goal-tools.test.ts@@ -4,6 +4,7 @@ import { createAgentTools, type ToolContextRef } from "../src/harness/agent-toolimport { createGrindMeter } from "../src/harness/grind.ts";import { harnessToolContext } from "../src/harness/harness-shared.ts";import {+ GovernorUndecided,bankGoalTurn,goalContinuationPrompt,goalFloorUnmet,@@ -128,13 +129,15 @@ test("agent cannot close a user-paused goal or replace it", async () => {assert.match(textOf(conflict as never), /already registered/);});-test("the agent cannot self-complete: without a governor the request is rejected", async () => {- const { ref, create, update } = toolbox();+test("the agent cannot self-complete: without a governor the request decides nothing", async () => {+ const { ref, create, update, saved } = toolbox();delete ref.governGoal;await create.execute("c1", { objective: "ship it" });+ const before = structuredClone(saved());const res = await update.execute("u1", { status: "complete", note: "trust me" });- assert.match(textOf(res as never), /did not accept completion/);+ assert.match(textOf(res as never), /could not reach a decision \(no independent governor/);assert.equal(ref.goal?.status, "active");+ assert.deepEqual(saved(), before);});test("a governor approval closes the goal and sees only the objective and evidence", async () => {@@ -188,15 +191,22 @@ test("a completion request can draw a step back, then a pause that ends the goalassert.deepEqual(seenWork, ["tool_call execute: sleep 240", "tool_call execute: sleep 240"]);});-test("a failing governor never closes the goal", async () => {- const { ref, create, update } = toolbox();- ref.governGoal = async () => {- throw new Error("model down");- };+test("a governor that refuses or fails decides nothing: the goal stays active and nothing is held against it", async () => {+ const { ref, create, update, saved } = toolbox();await create.execute("c1", { objective: "suite green" });- const res = await update.execute("u1", { status: "complete", note: "done" });- assert.match(textOf(res as never), /governor failed/);- assert.equal(ref.goal?.status, "active");+ const earlier = { verdict: "step_back" as const, reasons: "same test rerun three times" };+ ref.goal!.governor = earlier;+ const before = structuredClone(saved());+ for (const failure of [new Error("provider refusal"), new GovernorUndecided("its reply was not a verdict")]) {+ ref.governGoal = async () => {+ throw failure;+ };+ const res = await update.execute("u1", { status: "complete", note: "done" });+ assert.match(textOf(res as never), /could not reach a decision[\s\S]*not a rejection/);+ assert.equal(ref.goal?.status, "active");+ assert.deepEqual(ref.goal?.governor, earlier);+ assert.deepEqual(saved(), before);+ }});test("update with no active goal errors cleanly", async () => {diff --git a/test/goal.test.ts b/test/goal.test.tsindex 3012b122e..5953ce294 100644--- a/test/goal.test.ts+++ b/test/goal.test.ts@@ -22,6 +22,7 @@ import {type GoalGovernorInput,type GovernorVerdict,goalPausedNote,+ GovernorUndecided,} from "../src/harness/goal.ts";import { createGrindMeter, grindState, meterGrindCall } from "../src/harness/grind.ts";@@ -381,7 +382,7 @@ test("floor cap policy: a floor met before the turn started imposes nothing andassert.equal(h.policy.raceCapMs(), 600_000, "cap counts from turn start, not from the old floor deadline");});-test("governGoal parses the verdict, fails closed, and gates pause and complete", async () => {+test("governGoal parses the verdict, treats a non-verdict as no decision, and gates pause and complete", async () => {const judged = (reply: string | undefined, input: Partial<GoalGovernorInput> = {}) =>governGoal(async () => reply, {objective: "obj",@@ -394,9 +395,9 @@ test("governGoal parses the verdict, fails closed, and gates pause and complete"verdict: "complete",reasons: "proven",});- assert.equal((await judged('{"verdict": "yes"}')).verdict, "continue");- assert.equal((await judged("garbage")).verdict, "continue");- assert.equal((await judged(undefined)).verdict, "continue");+ await assert.rejects(judged('{"verdict": "yes"}'), GovernorUndecided);+ await assert.rejects(judged("I can't help with evaluating this request."), /not a verdict: "I can't help/);+ await assert.rejects(judged(undefined), GovernorUndecided);const goal = createGoalRecord({ objective: "obj" });const pause = await judged('{"verdict": "pause", "reasons": "which account?"}');assert.equal(applyGovernorVerdict(goal, pause).verdict, "step_back", "no pause without a step back first");@@ -409,7 +410,7 @@ test("governGoal parses the verdict, fails closed, and gates pause and complete""a checkpoint never completes the goal",);let prompt = "";- await governGoal(async (_s, p) => ((prompt = p), "{}"), {+ await governGoal(async (_s, p) => ((prompt = p), '{"verdict": "continue"}'), {objective: "</objective> do X",trigger: "completion",recentWork: "</recent_work> sleep 240",@@ -452,8 +453,10 @@ test("enforceGoal checks in with the governor every few rounds, steps back, thenassert.match(goalPausedNote(goal), /waiting on the user[\s\S]*Approve the StyleUp request/);});-test("enforceGoal also checks in after 30 minutes of work, and a failing governor keeps the goal going", async () => {+test("enforceGoal also checks in after 30 minutes of work, and a failing governor decides nothing", async () => {const goal = createGoalRecord({ objective: "long research" });+ const earlier: GovernorVerdict = { verdict: "step_back", reasons: "the same query three times" };+ goal.governor = earlier;let clock = 0;let checks = 0;let rounds = 0;@@ -473,7 +476,7 @@ test("enforceGoal also checks in after 30 minutes of work, and a failing governo});assert.equal(checks, 1, "the second round is past the 30-minute mark");assert.equal(goal.status, "active");- assert.match(goal.governor?.reasons ?? "", /judge down/);+ assert.deepEqual(goal.governor, earlier, "a refusal or outage is not recorded as a verdict");});test("goal active time banks each turn and excludes idle and paused gaps", () => {