Multiplayer agent harness for work.
Classify every turn and give rules the turn context (#2334)
Joshua Franceb47511a70bd4
diff --git a/src/decisions/hooks.ts b/src/decisions/hooks.tsindex 8c3e9d7b3..2a532c9dc 100644--- a/src/decisions/hooks.ts+++ b/src/decisions/hooks.ts@@ -4,13 +4,30 @@ import type { DecisionStore } from "./store.ts";const HOOK_TIMEOUT_MS = 5_000;-export async function routeTurnRuntime<R extends object>(- requested: R | undefined,- continueTurn: boolean | undefined,- classify: () => Promise<{ runtime?: R } | undefined>,-): Promise<R | undefined> {- if (continueTurn) return requested;- return (await classify())?.runtime ?? requested;+export function turnStartState(turn: {+ input: string;+ surfaceName?: string;+ runtimePurpose?: string;+ runtime?: { harnessId?: string; modelId?: string };+ runtimePinned?: boolean;+ continueTurn?: boolean;+ scopeLabel: string;+}): string {+ const requested = turn.runtime?.modelId+ ? [turn.runtime.harnessId, turn.runtime.modelId].filter(Boolean).join("/")+ : "none (org default applies)";+ return [+ "Turn context:",+ `- surface: ${turn.surfaceName ?? "unknown"}`,+ `- purpose: ${turn.runtimePurpose ?? "interactive"}`,+ `- scope: ${turn.scopeLabel.split(":")[0]}`,+ `- requested runtime: ${requested}`,+ `- model account: ${turn.runtimePinned ? "the user's own provider account, which serves only the requested runtime" : "company"}`,+ `- continued turn: ${turn.continueTurn ? "yes, resuming after a recorded tool result" : "no"}`,+ "",+ "Latest message:",+ turn.input,+ ].join("\n");}export interface DecisionTurn {diff --git a/src/wiring.ts b/src/wiring.tsindex 5cf544c20..61d60491f 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -3,7 +3,7 @@ import { resolveEmbeddingModel } from "./model/pi-models.ts";import { createPostgresTeamStore, type TeamStore } from "./teams/teams.ts";import { brokerCredentialAuthHeader } from "./api/credential-broker.ts";import type { DecisionBackend } from "./decisions/decisions.ts";-import { createDecisionHooks, routeTurnRuntime } from "./decisions/hooks.ts";+import { createDecisionHooks, turnStartState } from "./decisions/hooks.ts";import { createDecisionBackend } from "./decisions/providers.ts";import { createPostgresDecisionStore, type DecisionStore } from "./decisions/store.ts";import type { SlackSessionStatusState } from "./slack/session-status.ts";@@ -1504,11 +1504,6 @@ export function buildApp(};const harness = createHarnessRouter(adapters, adapters.get(fallbackHarness)!, async (input) => {await refreshModels();- if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {- if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))- throw new Error(`Unsupported model: ${input.runtime.modelId}`);- return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };- }const resolve = (requested: typeof input.runtime) =>resolveRuntimeChoiceDurable(configStore,@@ -1519,17 +1514,22 @@ export function buildApp(hydrateModelCatalog,input.runtimePurpose,);- return resolve(- await routeTurnRuntime(input.runtime, input.continueTurn, () =>- decide("turn_start", input.input, {- ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),- scopeLabel: input.scopeLabel,- sessionId: input.session.id,- ...(input.runId ? { runId: input.runId } : {}),- ...(input.cancel ? { cancel: input.cancel } : {}),- }),- ),- );+ const routed = (+ await decide("turn_start", turnStartState(input), {+ ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),+ scopeLabel: input.scopeLabel,+ sessionId: input.session.id,+ ...(input.runId ? { runId: input.runId } : {}),+ ...(input.cancel ? { cancel: input.cancel } : {}),+ })+ )?.runtime;+ if (routed) return resolve(routed);+ if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {+ if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))+ throw new Error(`Unsupported model: ${input.runtime.modelId}`);+ return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };+ }+ return resolve(input.runtime);});if (config.securityScreen === "off" || config.securityScreenClassifier !== "model") {diff --git a/test/decision-rules.test.ts b/test/decision-rules.test.tsindex c1ef6218c..e55395124 100644--- a/test/decision-rules.test.ts+++ b/test/decision-rules.test.ts@@ -11,7 +11,7 @@ import {type DecisionProvider,} from "../src/decisions/decisions.ts";import { createDecisionBackend } from "../src/decisions/providers.ts";-import { routeTurnRuntime } from "../src/decisions/hooks.ts";+import { turnStartState } from "../src/decisions/hooks.ts";const PROVIDERS: DecisionProvider[] = [{ id: "gw", protocol: "chat-completions" }];const MODELS = new Set(["m-deep"]);@@ -404,17 +404,24 @@ test("decision hooks load active rules for the hook, decide, and audit every ver);});-test("turn_start routing also applies when the turn already names a model", async () => {- const web = { harnessId: "claude", modelId: "claude-opus", effortLevel: "max" };- const routed = { modelId: "m-deep" };- assert.deepEqual(await routeTurnRuntime(web, false, async () => ({ runtime: routed })), routed);- assert.deepEqual(await routeTurnRuntime(web, false, async () => undefined), web);- assert.deepEqual(await routeTurnRuntime(undefined, false, async () => ({})), undefined);- let asked = false;- const kept = await routeTurnRuntime(web, true, async () => {- asked = true;- return { runtime: routed };+test("turn_start rules see the turn's context alongside the message", () => {+ const state = turnStartState({+ input: "Audit our session cookies",+ surfaceName: "web",+ runtime: { harnessId: "claude", modelId: "claude-opus" },+ runtimePinned: true,+ continueTurn: true,+ scopeLabel: "personal:[email protected]",});- assert.equal(kept, web);- assert.equal(asked, false, "a continued turn is never reclassified");+ for (const line of [+ "- surface: web",+ "- purpose: interactive",+ "- scope: personal",+ "- requested runtime: claude/claude-opus",+ "- model account: the user's own provider account",+ "- continued turn: yes",+ ])+ assert.ok(state.includes(line), line);+ assert.ok(state.endsWith("Latest message:\nAudit our session cookies"));+ assert.ok(turnStartState({ input: "hi", scopeLabel: "org:x" }).includes("- requested runtime: none"));});