Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Classify every turn and give rules the turn context (#2334)

Joshua Franceb47511a70bd4
diff --git a/src/decisions/hooks.ts b/src/decisions/hooks.ts
index 8c3e9d7b3..2a532c9dc 100644
--- a/src/decisions/hooks.ts
+++ b/src/decisions/hooks.ts
@@ -4,13 +4,30 @@ import type { DecisionStore } from "./store.ts";
const HOOK_TIMEOUT_MS = 5_000;
-export async function routeTurnRuntime<R extends object>(
- requested: R | undefined,
- continueTurn: boolean | undefined,
- classify: () => Promise<{ runtime?: R } | undefined>,
-): Promise<R | undefined> {
- if (continueTurn) return requested;
- return (await classify())?.runtime ?? requested;
+export function turnStartState(turn: {
+ input: string;
+ surfaceName?: string;
+ runtimePurpose?: string;
+ runtime?: { harnessId?: string; modelId?: string };
+ runtimePinned?: boolean;
+ continueTurn?: boolean;
+ scopeLabel: string;
+}): string {
+ const requested = turn.runtime?.modelId
+ ? [turn.runtime.harnessId, turn.runtime.modelId].filter(Boolean).join("/")
+ : "none (org default applies)";
+ return [
+ "Turn context:",
+ `- surface: ${turn.surfaceName ?? "unknown"}`,
+ `- purpose: ${turn.runtimePurpose ?? "interactive"}`,
+ `- scope: ${turn.scopeLabel.split(":")[0]}`,
+ `- requested runtime: ${requested}`,
+ `- model account: ${turn.runtimePinned ? "the user's own provider account, which serves only the requested runtime" : "company"}`,
+ `- continued turn: ${turn.continueTurn ? "yes, resuming after a recorded tool result" : "no"}`,
+ "",
+ "Latest message:",
+ turn.input,
+ ].join("\n");
}
export interface DecisionTurn {
diff --git a/src/wiring.ts b/src/wiring.ts
index 5cf544c20..61d60491f 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -3,7 +3,7 @@ import { resolveEmbeddingModel } from "./model/pi-models.ts";
import { createPostgresTeamStore, type TeamStore } from "./teams/teams.ts";
import { brokerCredentialAuthHeader } from "./api/credential-broker.ts";
import type { DecisionBackend } from "./decisions/decisions.ts";
-import { createDecisionHooks, routeTurnRuntime } from "./decisions/hooks.ts";
+import { createDecisionHooks, turnStartState } from "./decisions/hooks.ts";
import { createDecisionBackend } from "./decisions/providers.ts";
import { createPostgresDecisionStore, type DecisionStore } from "./decisions/store.ts";
import type { SlackSessionStatusState } from "./slack/session-status.ts";
@@ -1504,11 +1504,6 @@ export function buildApp(
};
const harness = createHarnessRouter(adapters, adapters.get(fallbackHarness)!, async (input) => {
await refreshModels();
- if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {
- if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))
- throw new Error(`Unsupported model: ${input.runtime.modelId}`);
- return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };
- }
const resolve = (requested: typeof input.runtime) =>
resolveRuntimeChoiceDurable(
configStore,
@@ -1519,17 +1514,22 @@ export function buildApp(
hydrateModelCatalog,
input.runtimePurpose,
);
- return resolve(
- await routeTurnRuntime(input.runtime, input.continueTurn, () =>
- decide("turn_start", input.input, {
- ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),
- scopeLabel: input.scopeLabel,
- sessionId: input.session.id,
- ...(input.runId ? { runId: input.runId } : {}),
- ...(input.cancel ? { cancel: input.cancel } : {}),
- }),
- ),
- );
+ const routed = (
+ await decide("turn_start", turnStartState(input), {
+ ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),
+ scopeLabel: input.scopeLabel,
+ sessionId: input.session.id,
+ ...(input.runId ? { runId: input.runId } : {}),
+ ...(input.cancel ? { cancel: input.cancel } : {}),
+ })
+ )?.runtime;
+ if (routed) return resolve(routed);
+ if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {
+ if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))
+ throw new Error(`Unsupported model: ${input.runtime.modelId}`);
+ return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };
+ }
+ return resolve(input.runtime);
});
if (config.securityScreen === "off" || config.securityScreenClassifier !== "model") {
diff --git a/test/decision-rules.test.ts b/test/decision-rules.test.ts
index c1ef6218c..e55395124 100644
--- a/test/decision-rules.test.ts
+++ b/test/decision-rules.test.ts
@@ -11,7 +11,7 @@ import {
type DecisionProvider,
} from "../src/decisions/decisions.ts";
import { createDecisionBackend } from "../src/decisions/providers.ts";
-import { routeTurnRuntime } from "../src/decisions/hooks.ts";
+import { turnStartState } from "../src/decisions/hooks.ts";
const PROVIDERS: DecisionProvider[] = [{ id: "gw", protocol: "chat-completions" }];
const MODELS = new Set(["m-deep"]);
@@ -404,17 +404,24 @@ test("decision hooks load active rules for the hook, decide, and audit every ver
);
});
-test("turn_start routing also applies when the turn already names a model", async () => {
- const web = { harnessId: "claude", modelId: "claude-opus", effortLevel: "max" };
- const routed = { modelId: "m-deep" };
- assert.deepEqual(await routeTurnRuntime(web, false, async () => ({ runtime: routed })), routed);
- assert.deepEqual(await routeTurnRuntime(web, false, async () => undefined), web);
- assert.deepEqual(await routeTurnRuntime(undefined, false, async () => ({})), undefined);
- let asked = false;
- const kept = await routeTurnRuntime(web, true, async () => {
- asked = true;
- return { runtime: routed };
+test("turn_start rules see the turn's context alongside the message", () => {
+ const state = turnStartState({
+ input: "Audit our session cookies",
+ surfaceName: "web",
+ runtime: { harnessId: "claude", modelId: "claude-opus" },
+ runtimePinned: true,
+ continueTurn: true,
+ scopeLabel: "personal:[email protected]",
});
- assert.equal(kept, web);
- assert.equal(asked, false, "a continued turn is never reclassified");
+ for (const line of [
+ "- surface: web",
+ "- purpose: interactive",
+ "- scope: personal",
+ "- requested runtime: claude/claude-opus",
+ "- model account: the user's own provider account",
+ "- continued turn: yes",
+ ])
+ assert.ok(state.includes(line), line);
+ assert.ok(state.endsWith("Latest message:\nAudit our session cookies"));
+ assert.ok(turnStartState({ input: "hi", scopeLabel: "org:x" }).includes("- requested runtime: none"));
});