Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Let the server own a chat's model and show routed turns in the picker (#2354)

Joshua France39ae2b4baad8
diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.ts
index 81fb0ccde..157cc5890 100644
--- a/plugins/web-ui/src/chat.ts
+++ b/plugins/web-ui/src/chat.ts
@@ -115,8 +115,10 @@ import {
type SubagentMailRef,
type ToolActivity,
type TurnOptions,
+ type TurnRuntime,
type WorkBlock,
fileContentUrl,
+ switchedDefault,
} from "./core-bridge";
import {
buildTimeline,
@@ -293,6 +295,7 @@ export function createChatSurface(
rememberedContextName: null as string | null,
liveWork: null as WorkBlock | null,
goal: null as GoalStripState | null,
+ turnRuntime: null as TurnRuntime | null,
pendingSend: null as string | null,
normalStreamFn: null as Agent["streamFn"] | null,
onWork: null as ((work: WorkBlock) => void) | null,
@@ -549,6 +552,7 @@ export function createChatSurface(
chatState.loadingEarlier = false;
chatState.pins = [];
chatState.goal = null;
+ chatState.turnRuntime = null;
chatState.host = document.createElement("div");
chatState.host.className = "custom-chat";
@@ -688,16 +692,21 @@ export function createChatSurface(
function currentTurnOptions(): TurnOptions {
const selected = ctx.composer.currentModelOption();
if (!selected) throw new Error("No model is available");
+ const target = { scopeId: chatState.scopeId, channelName: chatState.contextName };
+ if (ctx.composer.followsDefault()) return target;
const harness = selected.harnessId;
return {
...(harnessSupportsEffort(harness) ? { effortLevel: ctx.composer.state.effortLevel } : {}),
...(harnessSupportsFastMode(harness) ? { fastMode: ctx.composer.state.fastMode } : {}),
harness,
- scopeId: chatState.scopeId,
- channelName: chatState.contextName,
+ ...target,
};
}
+ function turnRuntime(): TurnRuntime | null {
+ return chatState.turnRuntime;
+ }
+
function inheritedHeader(): TemplateResult | typeof nothing {
const origin = chatState.forkSession
? forkOriginDetails(chatState.forkSession, chatState.inheritedLoaded ? chatState.inheritedMessages.length : 0)
@@ -893,6 +902,7 @@ export function createChatSurface(
return drawActiveChat(agent);
chatState.pins = page.pins ?? [];
chatState.goal = page.goal ?? null;
+ if (page.runtime) chatState.turnRuntime = page.runtime;
const split = inheritedTranscript(chatState.forkSession ?? {}, page.entries ?? []);
const messages = entriesToMessages(split.current, transcriptModel());
const refreshedInherited = inheritedRefreshEntries(
@@ -917,6 +927,15 @@ export function createChatSurface(
return (work: WorkBlock) => {
if (agent !== chatState.agent) return;
appendConsumedSteers(agent.state.messages, work);
+ if (
+ work.runtime &&
+ switchedDefault(work.runtime) &&
+ (work.runtime.harnessId !== chatState.turnRuntime?.harnessId ||
+ work.runtime.modelId !== chatState.turnRuntime.modelId)
+ )
+ void ctx.composer.followScopeDefault(agent);
+ if (work.runtime) chatState.turnRuntime = work.runtime;
+ else if (work !== chatState.liveWork) chatState.turnRuntime = null;
chatState.liveWork = work;
syncWorkTicker();
scheduleStreamDraw(agent);
@@ -1122,6 +1141,7 @@ export function createChatSurface(
chatState.inheritedExpanded = false;
chatState.pins = [];
chatState.goal = null;
+ chatState.turnRuntime = null;
}
syncLocation();
@@ -1314,6 +1334,11 @@ export function createChatSurface(
`;
}
+ function setTurnRuntime(runtime: TurnRuntime | null): void {
+ chatState.turnRuntime = runtime;
+ if (chatState.agent) drawActiveChat(chatState.agent);
+ }
+
function setGoal(goal: GoalStripState | null): void {
chatState.goal = goal;
if (chatState.agent) drawActiveChat(chatState.agent);
@@ -3478,6 +3503,7 @@ export function createChatSurface(
stopLiveRun,
isStopping: () => runSlot.stopGeneration === runSlot.generation,
currentTurnOptions,
+ turnRuntime,
newChat,
teardown: teardownActiveChat,
resetChatState,
@@ -3506,6 +3532,7 @@ export function createChatSurface(
setTranscriptWindow,
setPins,
setGoal,
+ setTurnRuntime,
requestBackgroundPanel,
activePendingApprovals,
hasUnresolvedApproval,
diff --git a/plugins/web-ui/src/composer.ts b/plugins/web-ui/src/composer.ts
index 9d5f6a055..080b88d1c 100644
--- a/plugins/web-ui/src/composer.ts
+++ b/plugins/web-ui/src/composer.ts
@@ -29,6 +29,7 @@ import {
type CoreAttachment,
type PendingApproval,
type QueuedRun,
+ switchedDefault,
} from "./core-bridge";
import { errMessage } from "../../chassis/src/errors";
import { browserRenderableImage, fieldSelect } from "./ui";
@@ -349,6 +350,18 @@ export function createComposerSurface(ctx: ConvCtx, options: ComposerOptions = {
return modelOptionFor(picked ?? defaultModelValue(scopeKey()), scopeKey());
}
+ function followsDefault(): boolean {
+ const selected = currentModelOption();
+ const threadRef = ctx.chat.state.threadRef;
+ return (
+ !!selected &&
+ !(threadRef && threadModelPicks.has(threadRef)) &&
+ effortOverride === undefined &&
+ fastModeOverride === undefined &&
+ restoredLoadout?.value !== selected.value
+ );
+ }
+
async function refreshRuntimeSelection(scopeId: string | null, agent?: Agent, refresh = false): Promise<void> {
const request = ++runtimeRequest;
const actualScope = runtimeScopeKey(scopeId);
@@ -476,10 +489,7 @@ export function createComposerSurface(ctx: ConvCtx, options: ComposerOptions = {
effortSelectionRevision === effortRevision &&
fastSelectionRevision === fastRevision
) {
- if (ctx.chat.state.threadRef) forgetThreadPick(ctx.chat.state.threadRef);
- effortOverride = undefined;
- fastModeOverride = undefined;
- restoredLoadout = undefined;
+ clearChatSelection();
}
}
syncRuntimeSelection(agent);
@@ -555,7 +565,14 @@ export function createComposerSurface(ctx: ConvCtx, options: ComposerOptions = {
}
const compact = Boolean(ctx.pane) || isPhone();
- const runtimeControls = modelPicker.render(agent, selectedModel, inputBlocked);
+ const turnRuntime = ctx.chat.turnRuntime();
+ const routed =
+ turnRuntime?.change &&
+ !switchedDefault(turnRuntime) &&
+ (turnRuntime.harnessId !== selectedModel.harnessId || turnRuntime.modelId !== selectedModel.model.id)
+ ? turnRuntime
+ : undefined;
+ const runtimeControls = modelPicker.render(agent, selectedModel, inputBlocked, routed);
return html`
<form
class="composer-wrap ${compact ? "compact" : ""}"
@@ -1883,6 +1900,18 @@ export function createComposerSurface(ctx: ConvCtx, options: ComposerOptions = {
ctx.chat.drawActiveChat(agent);
}
+ function clearChatSelection(): void {
+ if (ctx.chat.state.threadRef) forgetThreadPick(ctx.chat.state.threadRef);
+ effortOverride = undefined;
+ fastModeOverride = undefined;
+ restoredLoadout = undefined;
+ }
+
+ async function followScopeDefault(agent: Agent): Promise<void> {
+ clearChatSelection();
+ await refreshRuntimeSelection(ctx.chat.state.scopeId, agent, true);
+ }
+
function selectModel(value: string, agent: Agent): void {
const option = getModelOptions(scopeKey()).find((candidate) => candidate.value === value);
if (!option) return;
@@ -2092,6 +2121,8 @@ export function createComposerSurface(ctx: ConvCtx, options: ComposerOptions = {
},
resizeComposer,
currentModelOption,
+ followsDefault,
+ followScopeDefault,
carryModelPick,
refreshRuntimeSelection,
onDragEnter,
diff --git a/plugins/web-ui/src/conv-types.ts b/plugins/web-ui/src/conv-types.ts
index 098617752..99f7c7c39 100644
--- a/plugins/web-ui/src/conv-types.ts
+++ b/plugins/web-ui/src/conv-types.ts
@@ -9,6 +9,7 @@ import type {
PendingApproval,
QueuedRun,
TurnOptions,
+ TurnRuntime,
entriesToMessages,
} from "./core-bridge";
import type { EffortLevel, ModelOption } from "./model-options";
@@ -74,6 +75,7 @@ export interface ChatSurface {
stopLiveRun(): Promise<void>;
isStopping(): boolean;
currentTurnOptions(): TurnOptions;
+ turnRuntime(): TurnRuntime | null;
newChat(context?: { scopeId: string; name: string | null }): string;
teardown(): void;
resetChatState(): void;
@@ -101,6 +103,7 @@ export interface ChatSurface {
setTranscriptWindow(anchorSeq: number | null, earlierCount: number, hasEarlier?: boolean): void;
setPins(pins: import("./core-bridge").SessionPin[]): void;
setGoal(goal: import("./goal-strip").GoalStripState | null): void;
+ setTurnRuntime(runtime: TurnRuntime | null): void;
requestBackgroundPanel(sessionId: string | null, threadRef: string | null): void;
activePendingApprovals(): PendingApproval[];
hasUnresolvedApproval(): boolean;
@@ -143,6 +146,8 @@ export interface ComposerSurface {
sendSuggestedPrompt(prompt: string, agent: Agent): Promise<void>;
resizeComposer(): void;
currentModelOption(): ModelOption | undefined;
+ followsDefault(): boolean;
+ followScopeDefault(agent: Agent): Promise<void>;
carryModelPick(fromThreadRef: string | null, toThreadRef: string): void;
refreshRuntimeSelection(scopeId: string | null, agent?: Agent, refresh?: boolean): Promise<void>;
onDragEnter(e: DragEvent): void;
diff --git a/plugins/web-ui/src/core-bridge.ts b/plugins/web-ui/src/core-bridge.ts
index 840b95ec5..372571148 100644
--- a/plugins/web-ui/src/core-bridge.ts
+++ b/plugins/web-ui/src/core-bridge.ts
@@ -350,6 +350,7 @@ export interface TranscriptPage {
earlierEntries?: number;
pins?: SessionPin[];
goal?: GoalStripState;
+ runtime?: TurnRuntime;
}
export async function fetchTranscript(
@@ -445,6 +446,18 @@ export interface WorkBlock {
activity: ToolActivity[];
pendingApprovals?: PendingApproval[];
goal?: GoalStripState;
+ runtime?: TurnRuntime;
+}
+
+export interface TurnRuntime {
+ harnessId: string;
+ modelId: string;
+ label?: string;
+ change?: { rule: string } | { agent: "task" | "scope" };
+}
+
+export function switchedDefault(runtime: TurnRuntime | null | undefined): boolean {
+ return !!runtime?.change && "agent" in runtime.change && runtime.change.agent === "scope";
}
export function workPausedForApproval(work: WorkBlock): boolean {
@@ -491,6 +504,7 @@ export type AssistantWork = AssistantMessage & {
export interface RunPoll {
status: "pending" | "running" | "done" | "failed";
goal?: GoalStripState;
+ runtime?: TurnRuntime;
input?: {
runId: string;
seq: number | null;
@@ -1099,15 +1113,14 @@ function turnRequestBody(
const { idempotencyKey, attachments = [] } = send;
const turnOptions = getTurnOptions?.() ?? {};
const thinkingLevel =
- !turnOptions.harness || harnessSupportsEffort(turnOptions.harness)
+ turnOptions.harness && harnessSupportsEffort(turnOptions.harness)
? (turnOptions.effortLevel ?? agent.state.thinkingLevel ?? defaultEffortForModel(model))
: undefined;
const timezone = browserTimezone();
return {
text,
threadRef,
- ...(turnOptions.harness ? { harness: turnOptions.harness } : {}),
- model: model.id,
+ ...(turnOptions.harness ? { harness: turnOptions.harness, model: model.id } : {}),
...(thinkingLevel ? { thinkingLevel } : {}),
...(typeof turnOptions.fastMode === "boolean" ? { fastMode: turnOptions.fastMode } : {}),
...(timezone ? { timezone } : {}),
@@ -1386,6 +1399,10 @@ function applyRun(
work.goal = run.goal;
notify?.();
}
+ if (work && run.runtime && JSON.stringify(run.runtime) !== JSON.stringify(work.runtime)) {
+ work.runtime = run.runtime;
+ notify?.();
+ }
const p = typeof run.partial === "string" ? run.partial : "";
const startOffset = run.partialStartOffset ?? 0;
if (p && startOffset + p.length >= st.startOffset + st.acc.length) {
diff --git a/plugins/web-ui/src/model-picker.ts b/plugins/web-ui/src/model-picker.ts
index 30d0f7f89..52ee78acc 100644
--- a/plugins/web-ui/src/model-picker.ts
+++ b/plugins/web-ui/src/model-picker.ts
@@ -38,6 +38,13 @@ function effortText(level: EffortLevel | string): TemplateResult | string {
return rank >= EFFORT_PEAK_FLOOR ? html`<span class="effort-peak">${label}</span>` : label;
}
+interface RoutedRuntime {
+ harnessId: string;
+ modelId: string;
+ label?: string;
+ change?: { rule: string } | { agent: string };
+}
+
export interface ModelCatalog {
options(): ModelOption[];
supportsFast(modelId: string): boolean;
@@ -455,7 +462,12 @@ export function createModelPicker<T>(bindings: ModelPickerBindings<T>) {
</div>`;
}
- function render(agent: T, choice: ModelOption | undefined, disabled: boolean): TemplateResult {
+ function render(
+ agent: T,
+ choice: ModelOption | undefined,
+ disabled: boolean,
+ routed?: RoutedRuntime,
+ ): TemplateResult {
const selected = choice ?? catalog.options()[0];
if (!selected) return html`<span class="context-model-status">No models available</span>`;
const open = composerState.openMenu === "loadout";
@@ -471,6 +483,24 @@ export function createModelPicker<T>(bindings: ModelPickerBindings<T>) {
const fastUnsupportedReason = modelSupportsFast ? "Not supported by this harness" : "Not supported by this model";
const fastOn = fastAvailable && effectiveFastMode();
const speedLabel = (ultrafastOn && ", Ultrafast") || (fastOn && ", Fast") || "";
+ const settingLabel = choice ? modelLabel(choice) : (bindings.emptyLabel ?? "Choose model");
+ const settingAria = choice
+ ? `Model: ${settingLabel}, ${effortLabel(composerState.effortLevel)} effort${speedLabel}`
+ : settingLabel;
+ const routedOption = routed
+ ? catalog.options().find((option) => option.harnessId === routed.harnessId && option.model.id === routed.modelId)
+ : undefined;
+ const routedName = routedOption ? modelLabel(routedOption) : (routed?.label ?? routed?.modelId ?? "");
+ const rule = routed?.change && "rule" in routed.change ? routed.change.rule : undefined;
+ const reason = rule ? `routed by ${rule}` : "switched for this request";
+ const triggerLabel = routed
+ ? `Model: ${routedName}, ${reason}${rule ? " for this turn" : ""}; your setting is ${settingLabel}`
+ : settingAria;
+ const routedTip = routed
+ ? `${rule ? `The ${rule} rule` : "The agent"} ran this ${rule ? "turn" : "request"} on ${routedName}. Your setting is ${settingLabel}.`
+ : "";
+ const effortSuffix = choice ? effortText(composerState.effortLevel) : "";
+ const suffix = routed ? reason : effortSuffix;
return html`<div
class="menu-control loadout-control"
data-align="left"
@@ -492,7 +522,8 @@ export function createModelPicker<T>(bindings: ModelPickerBindings<T>) {
class="menu-button loadout-button ${ultrafastOn ? "ultrafast-active" : ""}"
data-focus-key=${`${loadoutMenuId}-trigger`}
type="button"
- aria-label=${choice ? `Model: ${modelLabel(choice)}, ${effortLabel(composerState.effortLevel)} effort${speedLabel}` : (bindings.emptyLabel ?? "Choose model")}
+ aria-label=${triggerLabel}
+ ${tip(routedTip)}
aria-haspopup="menu"
aria-expanded=${open ? "true" : "false"}
aria-controls=${loadoutMenuId}
@@ -520,8 +551,8 @@ export function createModelPicker<T>(bindings: ModelPickerBindings<T>) {
placeLoadout();
}}
>
- <span class="menu-label">${choice ? modelLabel(choice) : (bindings.emptyLabel ?? "Choose model")}</span>
- ${choice ? html`<span class="menu-suffix">${effortText(composerState.effortLevel)}</span>` : nothing}
+ <span class="menu-label">${routed ? routedName : settingLabel}</span>
+ ${suffix ? html`<span class="menu-suffix">${suffix}</span>` : nothing}
${ultrafastOn ? html`<span class="loadout-ultrafast-badge">${icon(Zap, 12)} Ultrafast</span>` : nothing}
${fastOn ? html`<span class="loadout-bolt">${icon(Zap, 13)}</span>` : nothing}${icon(ChevronDown, 13)}
</button>
diff --git a/plugins/web-ui/src/sessions.ts b/plugins/web-ui/src/sessions.ts
index e56f7083c..62b5473fe 100644
--- a/plugins/web-ui/src/sessions.ts
+++ b/plugins/web-ui/src/sessions.ts
@@ -1859,5 +1859,6 @@ export async function openSessionInto(
}
conv.setPins(entriesRes.pins ?? []);
conv.setGoal(entriesRes.goal ?? null);
+ conv.setTurnRuntime(entriesRes.runtime ?? null);
renderList();
}
diff --git a/plugins/web-ui/test/composer-picker.test.ts b/plugins/web-ui/test/composer-picker.test.ts
index d599a37ed..afb021cd7 100644
--- a/plugins/web-ui/test/composer-picker.test.ts
+++ b/plugins/web-ui/test/composer-picker.test.ts
@@ -177,6 +177,7 @@ test("the personal-account picker preserves composer choices and saves context d
},
activePendingApprovals: () => [],
hasUnresolvedApproval: () => false,
+ turnRuntime: () => null,
hasLiveRun: () => false,
drawActiveChat: draw,
},
@@ -713,6 +714,45 @@ test("the personal-account picker preserves composer choices and saves context d
assert.equal(button('[aria-label="Ultrafast"]').getAttribute("aria-checked"), "false");
assert.equal(composer!.state.effortLevel, "high");
}
+ config.effective = { harnessId: "pi", modelId: "gamma" };
+ localStorage.removeItem("web-ui:loadout");
+ ctx.chat.state.threadRef = "web:tester:server-default";
+ await composer!.refreshRuntimeSelection(null, agent, true);
+ await mount();
+ assert.equal(composer!.followsDefault(), true, "a chat on the default leaves the model to the server");
+ const routed = { harnessId: "pi", modelId: "beta", label: "Beta", change: { rule: "hard-task" } };
+ (ctx.chat as unknown as { turnRuntime: () => unknown }).turnRuntime = () => routed;
+ draw();
+ assert.equal(host.querySelector(".loadout-button .menu-label")?.textContent?.trim(), "Beta");
+ assert.equal(
+ button(".loadout-button").getAttribute("aria-label"),
+ "Model: Beta, routed by hard-task for this turn; your setting is Gamma",
+ );
+ localStorage.setItem("web-ui:model-picks", JSON.stringify([["web:tester:picked", "pi:beta"]]));
+ window.dispatchEvent(new dom.window.StorageEvent("storage", { key: "web-ui:model-picks" }));
+ ctx.chat.state.threadRef = "web:tester:picked";
+ await mount();
+ assert.equal(composer!.followsDefault(), false, "an explicit pick is sent with each message");
+ assert.match(button(".loadout-button").getAttribute("aria-label")!, /^Model: Beta, /);
+ ctx.chat.state.threadRef = "web:tester:server-default";
+ await mount();
+ (ctx.chat as unknown as { turnRuntime: () => unknown }).turnRuntime = () => ({
+ harnessId: "pi",
+ modelId: "beta",
+ change: { agent: "task" },
+ });
+ draw();
+ assert.equal(
+ button(".loadout-button").getAttribute("aria-label"),
+ "Model: Beta, switched for this request; your setting is Gamma",
+ );
+ (ctx.chat as unknown as { turnRuntime: () => unknown }).turnRuntime = () => ({
+ harnessId: "pi",
+ modelId: "beta",
+ change: { agent: "scope" },
+ });
+ draw();
+ assert.match(button(".loadout-button").getAttribute("aria-label")!, /^Model: Gamma, /);
} finally {
resetContext?.();
composer?.dispose();
diff --git a/plugins/web-ui/test/composer-stopping.test.ts b/plugins/web-ui/test/composer-stopping.test.ts
index fbbd81997..bf654c057 100644
--- a/plugins/web-ui/test/composer-stopping.test.ts
+++ b/plugins/web-ui/test/composer-stopping.test.ts
@@ -97,6 +97,7 @@ test("stopping returns Send immediately and queues every submit path without int
},
activePendingApprovals: () => [],
hasUnresolvedApproval: () => false,
+ turnRuntime: () => null,
isStopping: () => stopping,
drawActiveChat: draw,
stopLiveRun: async () => {
diff --git a/plugins/web-ui/test/runtime-config-views.test.ts b/plugins/web-ui/test/runtime-config-views.test.ts
index 977edf218..8a963cf64 100644
--- a/plugins/web-ui/test/runtime-config-views.test.ts
+++ b/plugins/web-ui/test/runtime-config-views.test.ts
@@ -84,6 +84,7 @@ test("runtime defaults are shared while pane choices and editor drafts remain lo
state: { scopeId, threadRef: `thread-${i}`, sessionId: `session-${i}`, agent, resolvingApprovals: new Set() },
activePendingApprovals: () => [],
hasUnresolvedApproval: () => false,
+ turnRuntime: () => null,
isStopping: () => false,
drawActiveChat: () => render(composer.composerForm(agent), host),
},
diff --git a/plugins/web-ui/test/suggested-activities-ui.test.ts b/plugins/web-ui/test/suggested-activities-ui.test.ts
index 2e0596fe7..cf496e8aa 100644
--- a/plugins/web-ui/test/suggested-activities-ui.test.ts
+++ b/plugins/web-ui/test/suggested-activities-ui.test.ts
@@ -103,6 +103,7 @@ test("activity selection fills and persists an editable draft without sending or
},
activePendingApprovals: () => [],
hasUnresolvedApproval: () => false,
+ turnRuntime: () => null,
hasLiveRun: () => false,
isStopping: () => false,
drawActiveChat: draw,
diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.ts
index 2c2ef743a..a058cbecb 100644
--- a/src/api/app-sessions.ts
+++ b/src/api/app-sessions.ts
@@ -264,7 +264,7 @@ export function createSessionMethods(
if (!found) return null;
const session = { ...found, canContinue: await canContinueOnWeb(principalId, found) };
let limit = tailWindowLimit(window);
- const [initialRead, pinRecords, goals] = await Promise.all([
+ const [initialRead, pinRecords, goals, latestRuns] = await Promise.all([
transcripts.forViewer(sessionId, principalId, {
limit,
beforeSeq: window?.beforeSeq,
@@ -272,8 +272,10 @@ export function createSessionMethods(
}),
deps.sessions.listPins(sessionId),
deps.sessions.goals([sessionId]),
+ window?.beforeSeq === undefined ? deps.runs.latestForThreads([found.threadRef]) : undefined,
]);
const goal = openGoal(goals.get(sessionId));
+ const runtime = latestRuns?.get(found.threadRef)?.result?.runtime;
let read = initialRead;
let visible = read.entries;
while (limit !== undefined && read.earlier > 0 && !coversTailWindow(visible, window!.tailTurns!)) {
@@ -290,6 +292,7 @@ export function createSessionMethods(
...(earlier > 0 ? { earlierEntries: earlier } : {}),
...(pins.length ? { pins } : {}),
...(goal ? { goal: goalView(goal) } : {}),
+ ...(runtime ? { runtime } : {}),
};
},
diff --git a/src/api/app-turn.ts b/src/api/app-turn.ts
index 019d8b67b..1f6bd645e 100644
--- a/src/api/app-turn.ts
+++ b/src/api/app-turn.ts
@@ -721,6 +721,7 @@ export function createTurnMethods(
if (viewer && !(await viewerMayUseRun(run, viewer))) return null;
const partial = deps.turnStream?.snapshot(runId);
const goal = deps.turnStream?.goal(runId);
+ const runtime = deps.turnStream?.runtime(runId) ?? run.result?.runtime;
const firstBlock = deps.turnStream?.firstBlock(runId);
const surfacePosted = deps.turnStream?.surfacePosted(runId) ?? false;
const alive = run.status === "running" && !leaseLapsed(run, Date.now());
@@ -758,6 +759,7 @@ export function createTurnMethods(
startedAt: run.startedAt,
finishedAt: run.finishedAt,
...(goal ? { goal } : {}),
+ ...(runtime ? { runtime } : {}),
...(partial ? { partial, partialStartOffset: deps.turnStream!.startOffset(runId) } : {}),
...(firstBlock
? { firstBlock: firstBlock.text, ...(firstBlock.closed ? { firstBlockClosed: true } : {}) }
diff --git a/src/api/app-types.ts b/src/api/app-types.ts
index 229dccc65..0f4bcc65d 100644
--- a/src/api/app-types.ts
+++ b/src/api/app-types.ts
@@ -20,6 +20,7 @@ import type {
SessionEntry,
TurnRequest,
TurnResult,
+ TurnRuntime,
} from "../types.ts";
import type { OutgoingAttachment } from "../types.ts";
import type { SecurityScreenProbe } from "../security/security-screener.ts";
@@ -298,6 +299,7 @@ export interface App {
attachments?: Array<{ name: string; mimetype: string; sizeBytes: number }>;
};
goal?: GoalView;
+ runtime?: TurnRuntime;
partial?: string;
partialStartOffset?: number;
alive?: boolean;
@@ -344,6 +346,7 @@ export interface App {
earlierEntries?: number;
pins?: SessionPinView[];
goal?: GoalView;
+ runtime?: TurnRuntime;
} | null>;
canViewSessionSnapshot(
sessionId: string,
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index 5bc91affd..021692630 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -30,6 +30,7 @@ import type {
SessionEntry,
SessionType,
TurnResult,
+ TurnRuntime,
PendingApproval,
PendingApprovalRecord,
} from "../types.ts";
@@ -3020,6 +3021,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
let firstChunkAt: number | undefined;
let lastChunkAt: number | undefined;
const emittedEntries: SessionEntry[] = [];
+ let turnRuntime: TurnRuntime | undefined;
const syntheticPrompt = syntheticInput || partial || approvalReplay || !!releasedToolOutput;
failureUserPayload =
!syntheticPrompt && input.text.trim()
@@ -3150,14 +3152,16 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
return errMessage(error);
}
};
- const adoptRuntime = async (choice: RuntimeChoice) => {
+ let agentSwitch: "task" | "scope" | undefined;
+ const adoptRuntime = async (choice: RuntimeChoice, lifetime: "task" | "scope") => {
const error = await checkRuntimeAuth(choice);
if (error) throw new NonRetryableTurnError(error);
+ agentSwitch = lifetime;
({ userProviderKeys, userModelOverride, userHarnessOverride, claudeOauthToken } =
await loadRuntimeAuth(choice));
requestedRuntime = userModelOverride ? { ...choice, modelId: userModelOverride } : choice;
};
- if (restoredRuntime) await adoptRuntime(restoredRuntime);
+ if (restoredRuntime) await adoptRuntime(restoredRuntime.choice, restoredRuntime.lifetime);
if (automatedTurn && input.model && inputHarness) {
const error = await deps.validateScheduledRuntime?.(
scopeId,
@@ -3374,6 +3378,11 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
? { runtimePinned: true }
: {}),
runtimeActorId: actor.id,
+ onRuntime: (reported: TurnRuntime) => {
+ const runtime: TurnRuntime = agentSwitch ? { ...reported, change: { agent: agentSwitch } } : reported;
+ turnRuntime = runtime;
+ if (input.runId) deps.turnStream?.noteRuntime(input.runId, runtime);
+ },
...(runtimePurpose ? { runtimePurpose } : {}),
...(deps.runtime && input.runId
? {
@@ -3629,7 +3638,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
throw new NonRetryableTurnError("The task reached its wall-clock limit during runtime handoff");
const recovery = "context" in segment.runtimeHandoff;
if ("choice" in segment.runtimeHandoff) {
- await adoptRuntime(segment.runtimeHandoff.choice);
+ await adoptRuntime(segment.runtimeHandoff.choice, segment.runtimeHandoff.lifetime);
await deps.harness.turns.resetSession?.(session.id);
}
let resumedHistory = filterHistory(
@@ -3989,6 +3998,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
};
}
+ if (turnRuntime) finalResult = { ...finalResult, runtime: turnRuntime };
await fallbackTitleWrite;
if (input.background && finalResult.status !== "pending_approval") {
diff --git a/src/decisions/hooks.ts b/src/decisions/hooks.ts
index 2a532c9dc..b49f95ac3 100644
--- a/src/decisions/hooks.ts
+++ b/src/decisions/hooks.ts
@@ -1,6 +1,7 @@
import type { AuditLog } from "../audit/audit-log.ts";
-import { runHook, type DecisionBackend, type DecisionHook } from "./decisions.ts";
+import { runHook, type DecisionBackend, type DecisionHook, type DecisionResult } from "./decisions.ts";
import type { DecisionStore } from "./store.ts";
+import { swallowAs } from "../util/errors.ts";
const HOOK_TIMEOUT_MS = 5_000;
@@ -62,3 +63,17 @@ export function createDecisionHooks(store: DecisionStore | undefined, backend: D
return decisive;
};
}
+
+export async function routeTurnRuntime<C extends { harnessId: string; modelId: string }>(
+ decision: DecisionResult | undefined,
+ route: (runtime: NonNullable<DecisionResult["runtime"]>) => Promise<C>,
+ requested: () => Promise<C>,
+): Promise<{ choice: C; routedBy?: string }> {
+ if (!decision?.runtime) return { choice: await requested() };
+ const [choice, otherwise] = await Promise.all([
+ route(decision.runtime),
+ requested().catch(swallowAs("decisions: requested runtime", undefined)),
+ ]);
+ const changed = !!otherwise && (otherwise.harnessId !== choice.harnessId || otherwise.modelId !== choice.modelId);
+ return changed ? { choice, routedBy: decision.ruleId } : { choice };
+}
diff --git a/src/harness/harness-router.ts b/src/harness/harness-router.ts
index 48714039f..21bb08431 100644
--- a/src/harness/harness-router.ts
+++ b/src/harness/harness-router.ts
@@ -6,6 +6,7 @@ import {
isHarnessId,
modelSupportedByHarness,
resolveModel,
+ safeModelMetadata,
thinkingLevelsForHarness,
modelUnavailableReason,
type HarnessId,
@@ -273,7 +274,7 @@ export async function resolveRuntimeChoiceDurable(
export function createHarnessRouter(
adapters: ReadonlyMap<HarnessId, Harness>,
utility: Harness,
- resolve: (input: HarnessTurnInput) => RuntimeChoice | Promise<RuntimeChoice>,
+ resolve: (input: HarnessTurnInput, routedBy: (ruleId: string) => void) => RuntimeChoice | Promise<RuntimeChoice>,
): Harness {
const lastHarness = new Map<string, HarnessId>();
return {
@@ -288,7 +289,17 @@ export function createHarnessRouter(
tools: utility.tools,
turns: {
async runTurn(input) {
- const choice = await resolve(input);
+ let routedBy: string | undefined;
+ const choice = await resolve(input, (ruleId) => {
+ routedBy = ruleId;
+ });
+ const label = safeModelMetadata(choice.modelId)?.label;
+ input.onRuntime?.({
+ harnessId: choice.harnessId,
+ modelId: choice.modelId,
+ ...(label ? { label } : {}),
+ ...(routedBy ? { change: { rule: routedBy } } : {}),
+ });
const adapter = adapters.get(choice.harnessId);
if (!adapter) throw new Error(`harness ${choice.harnessId} is unavailable`);
const prior = lastHarness.get(input.session.id);
diff --git a/src/harness/harness.ts b/src/harness/harness.ts
index 48720bfad..624fa8af6 100644
--- a/src/harness/harness.ts
+++ b/src/harness/harness.ts
@@ -10,6 +10,7 @@ import type {
Session,
SessionEntry,
TurnRequest,
+ TurnRuntime,
} from "../types.ts";
import type { HarnessId } from "../model/pi-models.ts";
import type { GapPhases, GapWork, LlmCallUsage, LlmTransportMeta, NewEntry } from "../sessions/session-store.ts";
@@ -127,6 +128,7 @@ export interface HarnessTurnInput {
orgScopeId: ScopeId;
providerKeys?: ProviderKeys;
runtimePinned?: boolean;
+ onRuntime?(runtime: TurnRuntime): void;
claudeOauthToken?: string;
recordModelCall(rec: { model: string; inputTokens: number; entryCount: number }): void;
recordLlmRequest?(rec: HarnessLlmRequestRecord, signal?: AbortSignal): void | Promise<void>;
diff --git a/src/harness/runtime-recovery.ts b/src/harness/runtime-recovery.ts
index 66ce1f74c..d2fe8c505 100644
--- a/src/harness/runtime-recovery.ts
+++ b/src/harness/runtime-recovery.ts
@@ -7,7 +7,7 @@ export function recoveredRuntime(
entries: readonly SessionEntry[],
runId: string,
actorId: string,
-): RuntimeChoice | undefined {
+): { choice: RuntimeChoice; lifetime: "task" | "scope" } | undefined {
for (const entry of [...entries].reverse()) {
const p = entry.payload;
if (
@@ -27,7 +27,10 @@ export function recoveredRuntime(
(choice.effortLevel === undefined || typeof choice.effortLevel === "string") &&
(choice.fastMode === undefined || typeof choice.fastMode === "boolean")
)
- return choice as unknown as RuntimeChoice;
+ return {
+ choice: choice as unknown as RuntimeChoice,
+ lifetime: p.runtimeHandoff.lifetime === "scope" ? "scope" : "task",
+ };
}
return undefined;
}
diff --git a/src/runs/turn-stream.ts b/src/runs/turn-stream.ts
index 7161be428..5d8c3bc79 100644
--- a/src/runs/turn-stream.ts
+++ b/src/runs/turn-stream.ts
@@ -1,4 +1,5 @@
import type { GoalRecord } from "../harness/goal.ts";
+import type { TurnRuntime } from "../types.ts";
export interface GoalView {
objective: string;
@@ -17,6 +18,8 @@ export interface TurnStream {
noteToolCall(runId: string): void;
noteGoal(runId: string, goal: GoalView): void;
goal(runId: string): GoalView | null;
+ noteRuntime(runId: string, runtime: TurnRuntime): void;
+ runtime(runId: string): TurnRuntime | null;
firstBlock(runId: string): { text: string; closed: boolean } | null;
markSurfacePosted(runId: string): void;
surfacePosted(runId: string): boolean;
@@ -37,6 +40,7 @@ interface Entry {
text: string;
totalChars: number;
goal: GoalView | null;
+ runtime: TurnRuntime | null;
firstBlock: string;
firstBlockOpen: boolean;
firstBlockClosed: boolean;
@@ -63,6 +67,7 @@ function makeEntry(partial: Partial<Entry> = {}): Entry {
text: "",
totalChars: 0,
goal: null,
+ runtime: null,
firstBlock: "",
firstBlockOpen: true,
firstBlockClosed: false,
@@ -149,6 +154,15 @@ export function createTurnStream(opts: TurnStreamOptions = {}): TurnStream {
return runs.get(runId)?.goal ?? null;
},
+ noteRuntime(runId, runtime) {
+ ensure(runId).runtime = runtime;
+ opts.onChange?.(runId);
+ },
+
+ runtime(runId) {
+ return runs.get(runId)?.runtime ?? null;
+ },
+
firstBlock(runId) {
const entry = runs.get(runId);
if (!entry || !entry.firstBlock.trim()) return null;
diff --git a/src/types.ts b/src/types.ts
index aed0d9b67..b21a4805b 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -780,6 +780,13 @@ export interface CommandApprovalGrant {
approvalKey?: string;
}
+export interface TurnRuntime {
+ harnessId: string;
+ modelId: string;
+ label?: string;
+ change?: { rule: string } | { agent: "task" | "scope" };
+}
+
export interface TurnResult {
status: "ok" | "refused" | "failed" | "pending_approval" | "queued" | "silent" | "react";
sessionId?: string;
@@ -795,4 +802,5 @@ export interface TurnResult {
attachments?: OutgoingAttachment[];
sourceUserSeq?: number;
sourceAssistantEntrySeq?: number;
+ runtime?: TurnRuntime;
}
diff --git a/src/wiring.ts b/src/wiring.ts
index 1d3eb9d3d..4e34d7e92 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -3,7 +3,7 @@ import { resolveEmbeddingModel } from "./model/pi-models.ts";
import { createPostgresTeamStore, type TeamStore } from "./teams/teams.ts";
import { brokerCredentialAuthHeader } from "./api/credential-broker.ts";
import type { DecisionBackend } from "./decisions/decisions.ts";
-import { createDecisionHooks, turnStartState } from "./decisions/hooks.ts";
+import { createDecisionHooks, routeTurnRuntime, turnStartState } from "./decisions/hooks.ts";
import { createDecisionBackend } from "./decisions/providers.ts";
import { createPostgresDecisionStore, type DecisionStore } from "./decisions/store.ts";
import type { SlackSessionStatusState } from "./slack/session-status.ts";
@@ -1500,7 +1500,7 @@ export function buildApp(
if (!(await modelCredentials.availability()).openrouter) return undefined;
return selectableModelCatalog(overrides.modelCredentialFetch);
};
- const harness = createHarnessRouter(adapters, adapters.get(fallbackHarness)!, async (input) => {
+ const harness = createHarnessRouter(adapters, adapters.get(fallbackHarness)!, async (input, routedBy) => {
await refreshModels();
const resolve = (requested: typeof input.runtime) =>
resolveRuntimeChoiceDurable(
@@ -1512,22 +1512,23 @@ export function buildApp(
hydrateModelCatalog,
input.runtimePurpose,
);
- const routed = (
- await decide("turn_start", turnStartState(input), {
- ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),
- scopeLabel: input.scopeLabel,
- sessionId: input.session.id,
- ...(input.runId ? { runId: input.runId } : {}),
- ...(input.cancel ? { cancel: input.cancel } : {}),
- })
- )?.runtime;
- if (routed) return resolve(routed);
- if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {
- if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))
- throw new Error(`Unsupported model: ${input.runtime.modelId}`);
- return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };
- }
- return resolve(input.runtime);
+ const decision = await decide("turn_start", turnStartState(input), {
+ ...(input.runtimeActorId ? { actorId: input.runtimeActorId } : {}),
+ scopeLabel: input.scopeLabel,
+ sessionId: input.session.id,
+ ...(input.runId ? { runId: input.runId } : {}),
+ ...(input.cancel ? { cancel: input.cancel } : {}),
+ });
+ const routed = await routeTurnRuntime(decision, resolve, async () => {
+ if (input.runtimePinned && input.runtime?.harnessId && input.runtime.modelId) {
+ if (!modelSupportedByHarness(input.runtime.modelId, input.runtime.harnessId))
+ throw new Error(`Unsupported model: ${input.runtime.modelId}`);
+ return { ...input.runtime, harnessId: input.runtime.harnessId, modelId: input.runtime.modelId };
+ }
+ return resolve(input.runtime);
+ });
+ if (routed.routedBy) routedBy(routed.routedBy);
+ return routed.choice;
});
if (config.securityScreen === "off" || config.securityScreenClassifier !== "model") {
diff --git a/test/decision-rules.test.ts b/test/decision-rules.test.ts
index 4eb90b66f..13c1e61ee 100644
--- a/test/decision-rules.test.ts
+++ b/test/decision-rules.test.ts
@@ -431,3 +431,28 @@ test("turn_start rules see the turn's context alongside the message", () => {
assert.ok(state.endsWith("Latest message:\nAudit our session cookies"));
assert.ok(turnStartState({ input: "hi", scopeLabel: "org:x" }).includes("- requested runtime: none"));
});
+
+test("a turn_start rule only counts as routing the turn when it changes the runtime", async () => {
+ const { routeTurnRuntime } = await import("../src/decisions/hooks.ts");
+ const opus = { harnessId: "pi", modelId: "claude-opus-5-5" };
+ const haiku = { harnessId: "pi", modelId: "claude-haiku-5-5" };
+ const verdict = {
+ ruleId: "hard-task",
+ option: "reroute",
+ action: "use_runtime" as const,
+ runtime: { modelId: haiku.modelId },
+ probabilities: { reroute: 0.9 },
+ mode: "enforce" as const,
+ };
+ const route = async (runtime: { modelId?: string }) => (runtime.modelId === haiku.modelId ? haiku : opus);
+
+ assert.deepEqual(await routeTurnRuntime(verdict, route, async () => opus), { choice: haiku, routedBy: "hard-task" });
+ assert.deepEqual(await routeTurnRuntime(verdict, route, async () => haiku), { choice: haiku });
+ assert.deepEqual(await routeTurnRuntime(undefined, route, async () => opus), { choice: opus });
+ assert.deepEqual(
+ await routeTurnRuntime(verdict, route, async () => {
+ throw new Error("unsupported pinned model");
+ }),
+ { choice: haiku },
+ );
+});
diff --git a/test/harness-router-goal.test.ts b/test/harness-router-goal.test.ts
index 20c1607b1..d8b593336 100644
--- a/test/harness-router-goal.test.ts
+++ b/test/harness-router-goal.test.ts
@@ -290,3 +290,27 @@ test("the agent cannot cancel its goal from a keep-going round", async () => {
assert.match(refusals[0]!, /Only the user's own message can cancel/);
assert.equal(savedGoal(emitted)?.status, "paused", "only the stop paused it; it was never cancelled");
});
+
+test("the router reports the runtime each turn runs on, and the rule that routed it", async () => {
+ const reported: unknown[] = [];
+ const reply: Round = async (turn) => {
+ await turn.emit({ type: "assistant", payload: { text: "done" }, scopeLabel: turn.scopeLabel });
+ return { reply: "done" };
+ };
+ const routed = createHarnessRouter(
+ new Map([["claude", fakeAdapter(reply).harness]]),
+ createMockHarness(),
+ async (_input, routedBy) => {
+ routedBy("hard-task");
+ return { harnessId: "claude", modelId: "claude-haiku-5-5" };
+ },
+ );
+ await routed.turns.runTurn(stubTurn([], { onRuntime: (runtime) => reported.push(runtime) }));
+ await router(fakeAdapter(reply).harness).turns.runTurn(
+ stubTurn([], { onRuntime: (runtime) => reported.push(runtime) }),
+ );
+ assert.deepEqual(reported, [
+ { harnessId: "claude", modelId: "claude-haiku-5-5", label: "Haiku 5.5", change: { rule: "hard-task" } },
+ { harnessId: "claude", modelId: "claude-opus-5-5", label: "Opus 5.5" },
+ ]);
+});
diff --git a/test/runtime-purpose.test.ts b/test/runtime-purpose.test.ts
index 890290d6e..558f09c1b 100644
--- a/test/runtime-purpose.test.ts
+++ b/test/runtime-purpose.test.ts
@@ -158,7 +158,7 @@ test("runtime get and inherit use job execution defaults, and handoffs survive r
},
] as SessionEntry[];
const restored = recoveredRuntime(entries, "run", "alice");
- assert.deepEqual(resolveRuntimeChoice(config, ORG, SCOPE, fallback, restored, "cron"), {
+ assert.deepEqual(resolveRuntimeChoice(config, ORG, SCOPE, fallback, restored?.choice, "cron"), {
...category,
...explicit,
modelId: "gpt-6-sol",
diff --git a/test/runtime-tool.test.ts b/test/runtime-tool.test.ts
index adccbedca..f7f15a089 100644
--- a/test/runtime-tool.test.ts
+++ b/test/runtime-tool.test.ts
@@ -105,7 +105,7 @@ test("runtime recovery uses only durable decisions belonging to the same run and
entry("other", "alice", "claude-sonnet-5"),
entry("run", "bob", "claude-opus-5"),
];
- assert.equal(recoveredRuntime(entries, "run", "alice")?.modelId, "gpt-6-astra");
+ assert.equal(recoveredRuntime(entries, "run", "alice")?.choice.modelId, "gpt-6-astra");
assert.equal(recoveredRuntime(entries, "new", "alice"), undefined);
});
diff --git a/test/turn-runtime-status.test.ts b/test/turn-runtime-status.test.ts
new file mode 100644
index 000000000..bc22d8825
--- /dev/null
+++ b/test/turn-runtime-status.test.ts
@@ -0,0 +1,95 @@
+import "./support/auto-fake-sprites.ts";
+import { mock, test } from "node:test";
+import assert from "node:assert/strict";
+import * as mockHarness from "../src/harness/mock-harness.ts";
+import { testConfig } from "./support/test-config.ts";
+
+let started!: () => void;
+let release!: () => void;
+let handoffTo: string | undefined;
+mock.module("../src/harness/mock-harness.ts", {
+ namedExports: {
+ ...mockHarness,
+ createMockHarness: () => {
+ const harness = mockHarness.createMockHarness();
+ harness.turns.runTurn = async () => {
+ if (handoffTo) {
+ const modelId = handoffTo;
+ handoffTo = undefined;
+ return {
+ reply: "",
+ runtimeHandoff: { choice: { harnessId: "mock", modelId }, lifetime: "task" as const },
+ };
+ }
+ started();
+ await new Promise<void>((resolve) => {
+ release = resolve;
+ });
+ return { reply: "Done" };
+ };
+ return harness;
+ },
+ },
+});
+const { buildApp } = await import("../src/wiring.ts");
+
+test("a run reports the runtime its turn runs on while working and after it finishes", async () => {
+ const built = buildApp(testConfig({ workers: 1 }));
+ built.runtime.start();
+ try {
+ for (const model of [undefined, "claude-haiku-5-5"]) {
+ const ready = new Promise<void>((resolve) => {
+ started = resolve;
+ });
+ const queued = await built.app.turn({
+ surface: "web",
+ liveActor: true,
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: `runtime-${model ?? "default"}` },
+ text: "hello",
+ async: true,
+ ...(model ? { model } : {}),
+ });
+ await ready;
+ const live = (await built.app.getRun(queued.runId!))?.runtime;
+ assert.ok(live?.modelId);
+ if (model) assert.equal(live.modelId, model);
+ assert.equal(live.change, undefined);
+ release();
+ await built.runs.waitFor(queued.runId!, 5000);
+ assert.deepEqual((await built.runs.get(queued.runId!))?.result?.runtime, live);
+ }
+ } finally {
+ release?.();
+ await built.runtime.stop();
+ }
+});
+
+test("a runtime the agent switches to mid-turn is reported as its switch", async () => {
+ const built = buildApp(testConfig({ workers: 1 }));
+ built.runtime.start();
+ try {
+ const ready = new Promise<void>((resolve) => {
+ started = resolve;
+ });
+ handoffTo = "claude-haiku-5-5";
+ const queued = await built.app.turn({
+ surface: "web",
+ liveActor: true,
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "runtime-switch" },
+ text: "switch me to haiku",
+ async: true,
+ });
+ await ready;
+ const live = (await built.app.getRun(queued.runId!))?.runtime;
+ assert.equal(live?.modelId, "claude-haiku-5-5");
+ assert.deepEqual(live.change, { agent: "task" });
+ release();
+ await built.runs.waitFor(queued.runId!, 5000);
+ assert.deepEqual((await built.runs.get(queued.runId!))?.result?.runtime, live);
+ } finally {
+ release?.();
+ await built.runtime.stop();
+ }
+});