Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fix model selection with personal AI credentials (#1556)

Joshua Francebe5bbf67db31
diff --git a/plugins/web-ui/src/composer.ts b/plugins/web-ui/src/composer.ts
index e4b4d93bc..465001555 100644
--- a/plugins/web-ui/src/composer.ts
+++ b/plugins/web-ui/src/composer.ts
@@ -174,7 +174,10 @@ export function resyncModelSelection(): void {
}
export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
- const refreshAccount = () => ctx.chat.drawActiveChat();
+ const refreshAccount = () => {
+ loadoutRestored = false;
+ void refreshRuntimeSelection(ctx.chat.state.scopeId, ctx.chat.state.agent ?? undefined, true);
+ };
window.addEventListener("model-account-changed", refreshAccount);
let runtimeRequest = 0;
let runtimeIdentity = "";
@@ -485,7 +488,6 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
}
const compact = Boolean(ctx.pane) || isPhone();
- const showRuntimeControls = !appState.me?.individualModelAuth;
const runtimeControls = modelPicker.render(agent, selectedModel, inputBlocked);
return html`
<form
@@ -617,7 +619,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {
>
${icon(Paperclip, 18)}
</button>
- ${showRuntimeControls ? runtimeControls : nothing}
+ ${runtimeControls}
</div>
<div class="composer-right">${sendControls(agent)}</div>
</div>
diff --git a/plugins/web-ui/src/runtime-config-store.ts b/plugins/web-ui/src/runtime-config-store.ts
index 52c234590..f19ad9e13 100644
--- a/plugins/web-ui/src/runtime-config-store.ts
+++ b/plugins/web-ui/src/runtime-config-store.ts
@@ -14,6 +14,19 @@ type Entry = {
const FRESH_MS = 30_000;
const entries = new Map<string, Entry>();
let bootScope: string | null = null;
+let accountRevision = 0;
+
+export function invalidateRuntimeConfigs(): void {
+ accountRevision++;
+ for (const entry of entries.values()) {
+ entry.generation++;
+ entry.config = null;
+ entry.fetchedAt = 0;
+ entry.load = null;
+ }
+}
+
+if (typeof window !== "undefined") window.addEventListener("model-account-changed", invalidateRuntimeConfigs);
function entryFor(scopeId: string): Entry {
let entry = entries.get(scopeId);
@@ -87,9 +100,10 @@ export async function saveRuntimeConfig(scopeId: string, change: Change): Promis
// Invalidate old reads at enqueue time, including when the save ultimately fails.
++entry.generation;
entry.load = null;
+ const revision = accountRevision;
const save = (entry.write ?? Promise.resolve()).then(async () => {
const config = await updateRuntimeConfig(scopeId, change);
- publish(scopeId, entry, config);
+ if (revision === accountRevision) publish(scopeId, entry, config);
return config;
});
const settled = save.then(
diff --git a/plugins/web-ui/test/composer-picker.test.ts b/plugins/web-ui/test/composer-picker.test.ts
index b403efebd..059c556f1 100644
--- a/plugins/web-ui/test/composer-picker.test.ts
+++ b/plugins/web-ui/test/composer-picker.test.ts
@@ -25,7 +25,7 @@ function model(id: string, label: string, provider = "anthropic"): ModelMetadata
};
}
-test("the shared picker preserves composer choices and saves context defaults", async () => {
+test("the personal-account picker preserves composer choices and saves context defaults", async () => {
const dom = new JSDOM('<!doctype html><div id="app"></div><div id="composer"></div>', {
url: "http://localhost/web-ui/",
pretendToBeVisual: true,
@@ -48,6 +48,7 @@ test("the shared picker preserves composer choices and saves context defaults",
};
const updates: Record<string, unknown>[] = [];
let failNextGet = true;
+ let runtimeReads = 0;
let failNextPut = false;
let deferNextPut = false;
let pendingPut: Promise<void> | undefined;
@@ -78,6 +79,7 @@ test("the shared picker preserves composer choices and saves context defaults",
},
fetch: async (input: RequestInfo | URL, init?: RequestInit) => {
if (String(input).startsWith("/api/runtime-config") && init?.method !== "PUT") {
+ runtimeReads++;
if (failNextGet) {
failNextGet = false;
return Response.json({ error: "initial load failed" }, { status: 500 });
@@ -160,7 +162,7 @@ test("the shared picker preserves composer choices and saves context defaults",
const { appState } = await vite.ssrLoadModule("/src/shell-state.ts");
const { createComposerSurface } = await vite.ssrLoadModule("/src/composer.ts");
const { render } = await vite.ssrLoadModule("lit");
- appState.me = { user: "tester", org: "test" };
+ appState.me = { user: "tester", org: "test", individualModelAuth: true, modelAuthConnected: true };
const host = document.querySelector<HTMLElement>("#composer")!;
const agent = { state: { isStreaming: false, messages: [] } } as unknown as Agent;
const draw = (): void => render(composer!.composerForm(agent), host);
@@ -217,6 +219,14 @@ test("the shared picker preserves composer choices and saves context defaults",
await composer!.refreshRuntimeSelection(null, agent, true);
assert.equal(composer!.state.effortLevel, "high", "identical refresh preserves restored effort");
assert.equal(composer!.state.fastMode, true, "identical refresh preserves restored Fast");
+ const readsBeforeAccountChange = runtimeReads;
+ appState.me.individualModelAuth = false;
+ window.dispatchEvent(new dom.window.CustomEvent("model-account-changed"));
+ await tick();
+ assert.equal(runtimeReads, readsBeforeAccountChange + 1);
+ assert.ok(host.querySelector(".loadout-button"));
+ appState.me.individualModelAuth = true;
+ draw();
const siblingHost = document.createElement("section");
document.body.append(siblingHost);
const siblingAgent = { state: { isStreaming: false, messages: [] } } as unknown as Agent;
diff --git a/plugins/web-ui/test/composer-source.test.ts b/plugins/web-ui/test/composer-source.test.ts
index 58b1247b8..42d31cd6b 100644
--- a/plugins/web-ui/test/composer-source.test.ts
+++ b/plugins/web-ui/test/composer-source.test.ts
@@ -24,7 +24,7 @@ test("compact and full composers share one left-side picker with Fast inside its
const leftStart = composer.indexOf('class="composer-left"');
const rightStart = composer.indexOf('class="composer-right"');
assert.ok(leftStart >= 0 && rightStart > leftStart);
- assert.ok(composer.slice(leftStart, rightStart).includes("showRuntimeControls ? runtimeControls : nothing"));
+ assert.ok(composer.slice(leftStart, rightStart).includes("${runtimeControls}"));
assert.ok(/class="composer-right">\$\{sendControls\(agent\)\}<\/div>/.test(composer));
const loadout = picker.slice(picker.indexOf("function render"), picker.indexOf("function menuArrowKeys"));
assert.ok(/role="menuitemcheckbox"\s+aria-label="Fast"/.test(loadout));
diff --git a/plugins/web-ui/test/runtime-config-sync.test.ts b/plugins/web-ui/test/runtime-config-sync.test.ts
index 6e0bf4841..f1ac7bd0e 100644
--- a/plugins/web-ui/test/runtime-config-sync.test.ts
+++ b/plugins/web-ui/test/runtime-config-sync.test.ts
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
import { afterEach, test } from "node:test";
import {
getRuntimeConfig,
+ invalidateRuntimeConfigs,
loadRuntimeConfig,
saveRuntimeConfig,
seedRuntimeConfig,
@@ -176,3 +177,37 @@ test("cached configuration is revalidated when returning to a view after the fre
Date.now = now;
}
});
+
+test("account changes invalidate all scopes and prevent stale requests from republishing", async () => {
+ const scope = "scope:account-race";
+ const other = "scope:account-other";
+ const stale = deferred<Response>();
+ const old = runtimeConfig(scope);
+ seedRuntimeConfig(other, runtimeConfig(other));
+ let reads = 0;
+ globalThis.fetch = async () => {
+ reads++;
+ return stale.promise;
+ };
+ const pending = loadRuntimeConfig(scope);
+ invalidateRuntimeConfigs();
+ assert.equal(getRuntimeConfig(other), null);
+ const personal = runtimeConfig(scope, { effective: { harnessId: "pi", modelId: "personal" } });
+ globalThis.fetch = async () => {
+ reads++;
+ return Response.json(personal);
+ };
+ await loadRuntimeConfig(scope, true);
+ stale.resolve(Response.json(old));
+ await pending;
+ assert.equal(reads, 2);
+ assert.deepEqual(getRuntimeConfig(scope), personal);
+
+ const saving = deferred<Response>();
+ globalThis.fetch = async () => saving.promise;
+ const write = saveRuntimeConfig(scope, { modelId: "old-account" });
+ invalidateRuntimeConfigs();
+ saving.resolve(Response.json(old));
+ await write;
+ assert.equal(getRuntimeConfig(scope), null);
+});
diff --git a/src/api/app-turn.ts b/src/api/app-turn.ts
index 88c7f6ca8..f2bd12c59 100644
--- a/src/api/app-turn.ts
+++ b/src/api/app-turn.ts
@@ -1,3 +1,4 @@
+import { userRuntimeConfigBody } from "./runtime-config.ts";
import { isSubagentThreadRef } from "../sessions/session-syscalls.ts";
import type { Conversation, Principal, TurnRequest, TurnResult } from "../types.ts";
import { orgId as orgIdOf } from "../config.ts";
@@ -176,6 +177,8 @@ export function createTurnMethods(
? await deps.config.getModelAccountDurable(actor.id)
: "company";
const individualAuth = modelAccount !== "company";
+ let requestedModel = req.model;
+ let requestedHarness = req.harness;
if (req.surface === "web") {
const threadRef = req.conversation.threadRef;
const existing = await deps.sessions.getByThread(threadRef);
@@ -194,6 +197,20 @@ export function createTurnMethods(
harnessId: fallbackHarness,
modelId: defaultModelForHarness(fallbackHarness),
};
+ if (individualAuth && (req.model || req.harness)) {
+ const available = await userRuntimeConfigBody({ deps }, targetScope, actor.id);
+ const harness = req.harness ?? available.effective.harnessId;
+ const model = req.model ?? available.effective.modelId;
+ if (!available.modelsByHarness[harness]?.includes(model))
+ return { status: "refused", reason: "Your connected AI account cannot serve this model on that harness." };
+ const invalidModelOption = validateWebTurnModelOptions(
+ { ...req, model },
+ available.modelsByHarness[harness] ?? [],
+ );
+ if (invalidModelOption) return { status: "refused", reason: invalidModelOption };
+ requestedModel = model;
+ requestedHarness = harness;
+ }
if (!individualAuth) {
const configuredKeys = deps.providerKeys ??
deps.modelProviders ?? { anthropic: false, openai: false, openrouter: false };
@@ -304,8 +321,8 @@ export function createTurnMethods(
...(req.detectOpener ? { detectOpener: req.detectOpener } : {}),
...(req.attachments?.length ? { attachments: req.attachments } : {}),
...(req.inboundNotes?.length ? { inboundNotes: req.inboundNotes } : {}),
- ...(!individualAuth && req.harness ? { harness: req.harness } : {}),
- ...(!individualAuth && req.model ? { model: req.model } : {}),
+ ...((!individualAuth || req.surface === "web") && requestedHarness ? { harness: requestedHarness } : {}),
+ ...((!individualAuth || req.surface === "web") && requestedModel ? { model: requestedModel } : {}),
...turnModelOptions(req),
...(req.readOnly ? { readOnly: true } : {}),
...(privateRequest
diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.ts
index 8c9901dc8..0e6518d62 100644
--- a/src/api/routes/surface.ts
+++ b/src/api/routes/surface.ts
@@ -1,6 +1,6 @@
import { isSessionStatus } from "../../sessions/session-status.ts";
import { suggestedActivityRoutes } from "./suggested-activities.ts";
-import { runtimeFallback, runtimeConfigBody, webuiModelEnabled } from "../runtime-config.ts";
+import { runtimeFallback, userRuntimeConfigBody, webuiModelEnabled } from "../runtime-config.ts";
import { sessionSharingRoutes } from "./session-sharing.ts";
import type { Grant, ScopeId, Session } from "../../types.ts";
import { parseScopeId, scopeId as makeScopeId } from "../../types.ts";
@@ -1218,7 +1218,7 @@ async function getRuntimeConfig(ctx: ApiCtx): Promise<void> {
const target = await runtimeTarget(ctx);
if (!target) return sendJson(ctx.res, 403, { error: "forbidden" });
await ctx.deps.refreshModels?.();
- return sendJson(ctx.res, 200, await runtimeConfigBody(ctx, target.scope));
+ return sendJson(ctx.res, 200, await userRuntimeConfigBody(ctx, target.scope, target.actorId));
}
async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {
@@ -1262,6 +1262,11 @@ async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {
const fastMode = ctx.body.fastMode ?? false;
if (typeof fastMode !== "boolean") return sendJson(ctx.res, 400, { error: "fast_mode_invalid" });
const choice = { harnessId, modelId, effortLevel, fastMode: fastMode && fastModeModelIds().includes(modelId) };
+ if ((await config.getModelAccountDurable(target.actorId)) !== "company") {
+ const available = await userRuntimeConfigBody(ctx, target.scope, target.actorId);
+ if (!available.modelsByHarness[harnessId]?.includes(modelId))
+ return sendJson(ctx.res, 400, { error: "account_runtime_unavailable" });
+ }
await config.setRuntimeSelectionLatest(target.scope, choice);
}
audit(ctx.deps, {
@@ -1270,7 +1275,7 @@ async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {
resource: "runtime-config",
scopeLabel: target.scope,
});
- return sendJson(ctx.res, 200, await runtimeConfigBody(ctx, target.scope));
+ return sendJson(ctx.res, 200, await userRuntimeConfigBody(ctx, target.scope, target.actorId));
}
async function getChannelHeaderPin(ctx: ApiCtx): Promise<void> {
diff --git a/src/api/runtime-config.ts b/src/api/runtime-config.ts
index 3068f47ba..e50641075 100644
--- a/src/api/runtime-config.ts
+++ b/src/api/runtime-config.ts
@@ -1,7 +1,8 @@
+import { resolveIndividualAuthRouting } from "../core/individual-auth-routing.ts";
import { gatewayModelCatalog } from "../model/gateway-models.ts";
-import type { ServerDeps } from "./deps.ts";
+import type { AppDeps } from "./app-types.ts";
import type { ScopeId } from "../types.ts";
-import { orgScope } from "./routes/shared.ts";
+import { orgScope } from "../config.ts";
import {
defaultModelForHarness,
isHarnessId,
@@ -16,21 +17,24 @@ import {
thinkingLevelsForHarness,
harnessSupportsFastMode,
codexProviderModelId,
+ codexSubscriptionModelId,
type HarnessId,
} from "../model/pi-models.ts";
import { builtInModelCatalog, selectableCatalogForHarness, selectableModelCatalog } from "../model/model-catalog.ts";
import type { RuntimeChoice } from "../harness/harness.ts";
-export type RuntimeDeps = Pick<
- ServerDeps,
- | "config"
- | "harnessId"
- | "baseModelDefault"
- | "providerKeys"
- | "modelCredentials"
- | "modelCredentialFetch"
- | "refreshModels"
->;
+export type RuntimeDeps = Partial<
+ Pick<
+ AppDeps,
+ | "config"
+ | "harnessId"
+ | "providerKeys"
+ | "modelCredentials"
+ | "modelCredentialFetch"
+ | "refreshModels"
+ | "userModelCredentials"
+ >
+> & { baseModelDefault?: string };
export function runtimeFallback(ctx: { deps: RuntimeDeps }): { harnessId: HarnessId; modelId: string } {
const harnessId = isHarnessId(ctx.deps.harnessId) ? ctx.deps.harnessId : "pi";
@@ -42,12 +46,55 @@ export function runtimeFallback(ctx: { deps: RuntimeDeps }): { harnessId: Harnes
return { harnessId, modelId: ctx.deps.baseModelDefault ?? defaultModelForHarness(harnessId, undefined, providers) };
}
-export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: ScopeId) {
+export async function userRuntimeConfigBody(ctx: { deps: RuntimeDeps }, scope: ScopeId, actorId: string) {
+ const account = await ctx.deps.config?.getModelAccountDurable(actorId);
+ const store = ctx.deps.userModelCredentials;
+ if (!store || !account || account === "company") return runtimeConfigBody(ctx, scope);
+ const [anthropic, openai] = await Promise.all([
+ account === "openai" ? null : store.get(actorId, "anthropic"),
+ account === "anthropic" ? null : store.get(actorId, "openai"),
+ ]);
+ const snapshot = await runtimeConfigBody(ctx, scope, async (choice) => {
+ const route = resolveIndividualAuthRouting(anthropic, openai, choice.modelId, choice.harnessId);
+ return route?.harness === choice.harnessId && route.model === choice.modelId ? null : "account_runtime_unavailable";
+ });
+ const route = resolveIndividualAuthRouting(
+ anthropic,
+ openai,
+ snapshot.effective.modelId,
+ snapshot.effective.harnessId,
+ );
+ if (!route?.model || !snapshot.modelsByHarness[route.harness]?.includes(route.model)) return snapshot;
+ const { unavailableReason: _, ...available } = snapshot;
+ return {
+ ...available,
+ effective: {
+ ...snapshot.effective,
+ harnessId: route.harness,
+ modelId: route.model,
+ effortLevel: thinkingLevelsForHarness(route.harness).includes(snapshot.effective.effortLevel ?? "auto")
+ ? snapshot.effective.effortLevel
+ : "auto",
+ fastMode:
+ snapshot.effective.fastMode === true &&
+ harnessSupportsFastMode(route.harness) &&
+ fastModeModelIds().includes(route.model),
+ },
+ };
+}
+
+export async function runtimeConfigBody(
+ ctx: { deps: RuntimeDeps },
+ scope: ScopeId,
+ authorizeChoice?: (choice: RuntimeChoice) => Promise<string | null>,
+) {
+ if (authorizeChoice)
+ ctx = { deps: { ...ctx.deps, providerKeys: ALL_PROVIDERS_AVAILABLE, modelCredentials: undefined } };
const config = ctx.deps.config!;
const configuredKeys = ctx.deps.providerKeys ?? ALL_PROVIDERS_AVAILABLE;
const managedKeys = ctx.deps.modelCredentials ? await ctx.deps.modelCredentials.availability() : configuredKeys;
const fallback = runtimeFallback(ctx);
- const org = orgScope(ctx.deps);
+ const org = orgScope();
const approvedHarnesses = ((await config.getApprovedHarnessesDurable()) ?? [fallback.harnessId]).filter(isHarnessId);
const providersFor = (harnessId: string) => modelProviderAvailabilityFor(harnessId, configuredKeys, managedKeys);
const catalog =
@@ -108,6 +155,10 @@ export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: Scope
for (const choice of selected) {
if (
allowlist?.length !== 0 &&
+ (!authorizeChoice ||
+ !allowlist ||
+ choice.modelId === orgDefault.modelId ||
+ allowlist.includes(codexProviderModelId(choice.modelId))) &&
choice.harnessId === harnessId &&
modelSupportedByHarness(choice.modelId, harnessId) &&
!ids.includes(choice.modelId)
@@ -117,6 +168,21 @@ export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: Scope
return [harnessId, serviceableModelIds(ids, providersFor(harnessId))];
}),
);
+ if (authorizeChoice) {
+ const piModels = modelsByHarness.pi ?? [];
+ for (const id of piModels) {
+ const subscriptionId = codexSubscriptionModelId(id);
+ if (modelSupportedByHarness(subscriptionId, "pi") && !piModels.includes(subscriptionId))
+ piModels.push(subscriptionId);
+ }
+ for (const harnessId of approvedHarnesses) {
+ const candidates = modelsByHarness[harnessId] ?? [];
+ const allowed = await Promise.all(
+ candidates.map((modelId) => authorizeChoice({ harnessId, modelId, effortLevel: "auto", fastMode: false })),
+ );
+ modelsByHarness[harnessId] = candidates.filter((_, index) => !allowed[index]);
+ }
+ }
const advertisedModelIds = new Set(Object.values(modelsByHarness).flat());
const modelCatalog = Object.fromEntries(
[...advertisedModelIds].flatMap((id) => {
@@ -162,10 +228,10 @@ export function validateRuntimeChoice(choice: RuntimeChoice): string | null {
export async function webuiModelEnabled(ctx: { deps: RuntimeDeps }, modelId: string): Promise<boolean> {
modelId = codexProviderModelId(modelId);
const config = ctx.deps.config!;
- const picker = await config.getWebuiModelsDurable(orgScope(ctx.deps));
+ const picker = await config.getWebuiModelsDurable(orgScope());
if (picker == null || picker.includes(modelId)) return true;
if (picker.length === 0) return false;
- const org = orgScope(ctx.deps);
+ const org = orgScope();
const stored = await config.getRuntimeSelectionDurable(org);
const orgModel = stored?.modelId ?? (await config.getBaseModelOwnDurable(org)) ?? runtimeFallback(ctx).modelId;
return modelId === orgModel;
diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts
index c8e172824..09eac66fc 100644
--- a/src/core/orchestrator.ts
+++ b/src/core/orchestrator.ts
@@ -2958,8 +2958,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
const routing = resolveIndividualAuthRouting(
anthCred ?? null,
oaiCred ?? null,
- account === "personal" ? (runtime.modelId ?? input.model) : runtime.modelId,
- account === "personal" ? preferredHarness : runtime.harnessId,
+ account === "personal" || input.surface === "web" ? (runtime.modelId ?? input.model) : runtime.modelId,
+ account === "personal" || input.surface === "web" ? preferredHarness : runtime.harnessId,
);
if (routing?.kind === "apikey") {
userHarnessOverride = "pi";
@@ -3007,6 +3007,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {
};
let { userProviderKeys, userModelOverride, userHarnessOverride, claudeOauthToken, codexTurnAuth } =
await loadRuntimeAuth({});
+ if (
+ input.surface === "web" &&
+ userHarnessOverride &&
+ ((input.model && input.model !== userModelOverride) ||
+ (input.harness && input.harness !== userHarnessOverride))
+ )
+ throw new NonRetryableTurnError("Your connected AI account cannot serve this model on that harness.");
const effectiveModel = userModelOverride ?? input.model;
const effectiveHarness = userHarnessOverride ?? input.harness;
if (userHarnessOverride) {
diff --git a/src/harness/runtime-control.ts b/src/harness/runtime-control.ts
index d9001ef5c..2a2f32f43 100644
--- a/src/harness/runtime-control.ts
+++ b/src/harness/runtime-control.ts
@@ -9,14 +9,7 @@ import {
} from "../api/runtime-config.ts";
import { livePersonCapability } from "../api/artifact-share.ts";
import { parseScopeId } from "../types.ts";
-import {
- ALL_PROVIDERS_AVAILABLE,
- isHarnessId,
- thinkingLevelsForHarness,
- safeModelMetadata,
- modelSupportedByHarness,
- codexSubscriptionModelId,
-} from "../model/pi-models.ts";
+import { isHarnessId, thinkingLevelsForHarness } from "../model/pi-models.ts";
export function createRuntimeService(deps: RuntimeDeps, app: Pick<App, "authorizesCapabilityScope">): RuntimeService {
return async (claims, active, request, authorizeChoice, individualAuth, signal) => {
@@ -29,31 +22,7 @@ export function createRuntimeService(deps: RuntimeDeps, app: Pick<App, "authoriz
)
return { ok: false, error: "forbidden" };
await deps.refreshModels?.();
- const snapshot = await runtimeConfigBody(
- { deps: individualAuth ? { ...deps, providerKeys: ALL_PROVIDERS_AVAILABLE, modelCredentials: undefined } : deps },
- claims.scopeId,
- );
- if (individualAuth && authorizeChoice) {
- if (snapshot.approvedHarnesses.includes("pi")) {
- const piModels = snapshot.modelsByHarness.pi ?? [];
- for (const id of piModels) {
- const subscriptionId = codexSubscriptionModelId(id);
- if (subscriptionId === id) continue;
- if (modelSupportedByHarness(subscriptionId, "pi") && !piModels.includes(subscriptionId)) {
- piModels.push(subscriptionId);
- const metadata = safeModelMetadata(subscriptionId);
- if (metadata) snapshot.modelCatalog[subscriptionId] = metadata;
- }
- }
- }
- for (const harnessId of snapshot.approvedHarnesses) {
- const candidates = snapshot.modelsByHarness[harnessId] ?? [];
- const allowed = await Promise.all(
- candidates.map((modelId) => authorizeChoice({ harnessId, modelId, effortLevel: "auto", fastMode: false })),
- );
- snapshot.modelsByHarness[harnessId] = candidates.filter((_, index) => !allowed[index]);
- }
- }
+ const snapshot = await runtimeConfigBody({ deps }, claims.scopeId, individualAuth ? authorizeChoice : undefined);
if (request.action === "get")
return {
ok: true,
diff --git a/test/personal-model-account-execution.test.ts b/test/personal-model-account-execution.test.ts
index 5ed91ac32..7fbd6bdc4 100644
--- a/test/personal-model-account-execution.test.ts
+++ b/test/personal-model-account-execution.test.ts
@@ -82,3 +82,183 @@ test("disconnecting a queued personal account fails without invoking any main ha
await built.runtime.stop();
}
});
+
+for (const account of ["personal", "openai", "anthropic"] as const) {
+ test(`web ${account} account honors the chosen model, effort and fast mode`, async () => {
+ turns.length = 0;
+ const provider = account === "anthropic" ? "anthropic" : "openai";
+ const model = provider === "anthropic" ? "claude-sonnet-5" : "gpt-5.6-terra";
+ const built = buildApp(testConfig({ openaiApiKey: "company-openai" }));
+ built.config.setApprovedHarnesses(["pi"]);
+ await built.config.flushScope("org:default-org");
+ await built.userModelCredentials.setApiKey("U1", provider, `personal-${provider}`);
+ await built.config.setPersonalModelAuth("U1", true, account === "personal" ? undefined : account);
+ const submitted = await built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: `web:U1:picker-${account}` },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ model,
+ harness: "pi",
+ thinkingLevel: "high",
+ fastMode: true,
+ });
+ assert.ok(submitted.runId, JSON.stringify(submitted));
+ const queued = await built.runs.get(submitted.runId!);
+ assert.equal(queued?.request.model, model);
+ assert.equal(queued?.request.harness, "pi");
+ built.runtime.start();
+ try {
+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);
+ assert.equal(finished.status, "done", JSON.stringify(finished.result));
+ assert.equal(turns.length, 1);
+ assert.deepEqual(turns[0]!.providerKeys, { [provider]: `personal-${provider}` });
+ assert.deepEqual(turns[0]!.runtime, {
+ harnessId: "pi",
+ modelId: model,
+ effortLevel: "high",
+ fastMode: true,
+ });
+ } finally {
+ await built.runtime.stop();
+ }
+ });
+}
+
+test("personal web selections reject wrong providers, harnesses, policy exclusions and disconnected keys", async () => {
+ const built = buildApp(testConfig({ openaiApiKey: "company-openai", anthropicApiKey: "company-anthropic" }));
+ built.config.setApprovedHarnesses(["pi", "claude"]);
+ await built.config.flushScope("org:default-org");
+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");
+ await built.config.setPersonalModelAuth("U1", true, "openai");
+ const submit = (model: string, harness = "pi") =>
+ built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: `web:U1:invalid-${crypto.randomUUID()}` },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ model,
+ harness,
+ });
+ for (const [model, harness] of [
+ ["claude-sonnet-5", "pi"],
+ ["gpt-5.6-terra", "claude"],
+ ["unknown", "pi"],
+ ])
+ assert.equal((await submit(model!, harness!)).status, "refused");
+ built.config.setWebuiModels("org:default-org", ["gpt-5.6-sol"]);
+ await built.config.flushScope("org:default-org");
+ assert.equal((await submit("gpt-5.6-terra")).status, "refused");
+ await built.userModelCredentials.delete("U1", "openai");
+ assert.equal((await submit("gpt-5.6-sol")).status, "refused");
+});
+
+test("a queued web selection cannot fall back to another personal provider after disconnect", async () => {
+ turns.length = 0;
+ const built = buildApp(testConfig());
+ built.config.setApprovedHarnesses(["pi"]);
+ await built.config.flushScope("org:default-org");
+ await built.userModelCredentials.setApiKey("U1", "anthropic", "personal-anthropic");
+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");
+ await built.config.setPersonalModelAuth("U1", true);
+ const submitted = await built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "web:U1:disconnect-after-pick" },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ model: "gpt-5.6-terra",
+ harness: "pi",
+ });
+ assert.ok(submitted.runId, JSON.stringify(submitted));
+ await built.userModelCredentials.delete("U1", "openai");
+ built.runtime.start();
+ try {
+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);
+ assert.equal(finished.status, "failed");
+ assert.match(JSON.stringify(finished), /cannot serve this model/);
+ assert.equal(turns.length, 0);
+ } finally {
+ await built.runtime.stop();
+ }
+});
+
+test("web subscription selections use the namespaced model and personal OAuth on Pi", async () => {
+ turns.length = 0;
+ const built = buildApp(testConfig());
+ built.config.setApprovedHarnesses(["pi"]);
+ await built.config.flushScope("org:default-org");
+ await built.userModelCredentials.setOAuth("U1", "openai", {
+ accessToken: "personal-oauth",
+ expiresAt: Date.now() + 3_600_000,
+ });
+ await built.config.setPersonalModelAuth("U1", true, "openai");
+ const submitted = await built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "web:U1:oauth-picker" },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ model: "codex/gpt-5.6-terra",
+ harness: "pi",
+ thinkingLevel: "high",
+ fastMode: false,
+ });
+ assert.ok(submitted.runId, JSON.stringify(submitted));
+ built.runtime.start();
+ try {
+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);
+ assert.equal(finished.status, "done", JSON.stringify(finished.result));
+ assert.equal(turns.length, 1);
+ assert.deepEqual(turns[0]!.providerKeys, { "openai-codex": "personal-oauth" });
+ assert.deepEqual(turns[0]!.runtime, {
+ harnessId: "pi",
+ modelId: "codex/gpt-5.6-terra",
+ effortLevel: "high",
+ fastMode: false,
+ });
+ } finally {
+ await built.runtime.stop();
+ }
+});
+
+test("partial personal web choices queue the complete validated scoped runtime", async () => {
+ const built = buildApp(testConfig());
+ built.config.setApprovedHarnesses(["pi", "codex"]);
+ await built.config.setRuntimeSelectionLatest("org:default-org", { harnessId: "pi", modelId: "gpt-5.6-sol" });
+ await built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "pi", modelId: "gpt-5.6-terra" });
+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");
+ await built.config.setPersonalModelAuth("U1", true, "openai");
+ const submit = (choice: { harness?: string; model?: string }) =>
+ built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: `web:U1:partial-${crypto.randomUUID()}` },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ ...choice,
+ });
+ const first = await submit({ harness: "pi" });
+ assert.ok(first.runId, JSON.stringify(first));
+ const queued = await built.runs.get(first.runId!);
+ assert.equal(queued?.request.harness, "pi");
+ assert.equal(queued?.request.model, "gpt-5.6-terra");
+ await built.userModelCredentials.delete("U1", "openai");
+ await built.userModelCredentials.setOAuth("U1", "openai", {
+ accessToken: "personal-oauth",
+ expiresAt: Date.now() + 3_600_000,
+ });
+ await built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "codex", modelId: "gpt-5.6-sol" });
+ const second = await submit({ model: "gpt-5.6-terra" });
+ assert.ok(second.runId, JSON.stringify(second));
+ const oauth = await built.runs.get(second.runId!);
+ assert.equal(oauth?.request.harness, "codex");
+ assert.equal(oauth?.request.model, "gpt-5.6-terra");
+});
diff --git a/test/personal-runtime-picker.test.ts b/test/personal-runtime-picker.test.ts
new file mode 100644
index 000000000..289bffad3
--- /dev/null
+++ b/test/personal-runtime-picker.test.ts
@@ -0,0 +1,129 @@
+import "./support/auto-fake-sprites.ts";
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import type { AddressInfo } from "node:net";
+import { buildApp } from "../src/wiring.ts";
+import { createInsecureTestServer } from "../src/api/server.ts";
+import { testConfig } from "./support/test-config.ts";
+import type { userRuntimeConfigBody } from "../src/api/runtime-config.ts";
+import { resolveModel } from "../src/model/pi-models.ts";
+
+type Snapshot = Awaited<ReturnType<typeof userRuntimeConfigBody>>;
+
+async function setup() {
+ const built = buildApp(testConfig());
+ built.config.setApprovedHarnesses(["pi", "claude", "codex"]);
+ await built.config.flushScope("org:default-org");
+ const server = createInsecureTestServer(built.app, {
+ config: built.config,
+ userModelCredentials: built.userModelCredentials,
+ harnessId: "pi",
+ providerKeys: { anthropic: false, openai: false, openrouter: false },
+ });
+ server.listen(0);
+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;
+ const get = async (user = "U1", scope = `personal:${user}`) => {
+ const response = await fetch(`${base}/v1/runtime-config?principalId=${user}&scopeId=${encodeURIComponent(scope)}`);
+ assert.equal(response.status, 200);
+ return response.json() as Promise<Snapshot>;
+ };
+ const put = (choice: object) =>
+ fetch(`${base}/v1/runtime-config`, {
+ method: "PUT",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({ principalId: "U1", scopeId: "personal:U1", ...choice }),
+ });
+ return { built, get, put, close: () => new Promise<void>((r) => server.close(() => r())) };
+}
+
+test("personal API-key picker works without company keys and saves model, effort and Fast", async () => {
+ const s = await setup();
+ try {
+ await s.built.userModelCredentials.setApiKey("U1", "openai", "synthetic-openai");
+ await s.built.config.setPersonalModelAuth("U1", true, "openai");
+ const before = await s.get();
+ assert.ok(before.modelsByHarness.pi!.includes("gpt-5.6-terra"));
+ assert.ok(before.modelsByHarness.pi!.every((id: string) => resolveModel(id)?.provider === "openai"));
+ assert.deepEqual(before.modelsByHarness.claude, []);
+ assert.deepEqual(before.modelsByHarness.codex, []);
+ assert.ok(before.modelsByHarness.pi!.includes(before.effective.modelId));
+ assert.equal(before.unavailableReason, undefined);
+ assert.doesNotMatch(JSON.stringify(before), /synthetic-openai/);
+ const choice = { harnessId: "pi", modelId: "gpt-5.6-terra", effortLevel: "high", fastMode: true };
+ const saved = await s.put(choice);
+ assert.equal(saved.status, 200);
+ assert.deepEqual(((await saved.json()) as Snapshot).effective, choice);
+ assert.deepEqual((await s.get()).effective, choice);
+ assert.equal((await s.put({ ...choice, modelId: "claude-sonnet-5" })).status, 400);
+ await s.built.userModelCredentials.delete("U1", "openai");
+ assert.deepEqual((await s.get()).modelsByHarness.pi, []);
+ assert.equal((await s.put(choice)).status, 400);
+ } finally {
+ await s.close();
+ }
+});
+
+for (const provider of ["anthropic", "openai"] as const) {
+ test(`personal ${provider} OAuth picker advertises only compatible runtimes`, async () => {
+ const s = await setup();
+ try {
+ await s.built.userModelCredentials.setOAuth("U1", provider, {
+ accessToken: "synthetic-access",
+ refreshToken: "synthetic-refresh",
+ expiresAt: Date.now() + 3_600_000,
+ });
+ await s.built.config.setPersonalModelAuth("U1", true, provider);
+ const config = await s.get();
+ if (provider === "anthropic") {
+ assert.deepEqual(config.modelsByHarness.pi, []);
+ assert.ok(config.modelsByHarness.claude!.includes("claude-sonnet-5"));
+ assert.deepEqual(config.modelsByHarness.codex, []);
+ } else {
+ assert.ok(config.modelsByHarness.pi!.includes("codex/gpt-5.6-sol"));
+ assert.ok(!config.modelsByHarness.pi!.includes("gpt-5.6-sol"));
+ assert.ok(config.modelsByHarness.codex!.includes("gpt-5.6-sol"));
+ assert.deepEqual(config.modelsByHarness.claude, []);
+ }
+ assert.ok(config.modelsByHarness[config.effective.harnessId]!.includes(config.effective.modelId));
+ } finally {
+ await s.close();
+ }
+ });
+}
+
+test("personal picker keeps org restrictions and shared-scope caller isolation", async () => {
+ const s = await setup();
+ try {
+ await s.built.userModelCredentials.setApiKey("U1", "openai", "synthetic-openai");
+ await s.built.userModelCredentials.setApiKey("U2", "anthropic", "synthetic-anthropic");
+ await s.built.config.setPersonalModelAuth("U1", true, "openai");
+ await s.built.config.setPersonalModelAuth("U2", true, "anthropic");
+ await s.built.directory.replaceGroups([
+ { groupId: "room", principalId: "U1" },
+ { groupId: "room", principalId: "U2" },
+ ]);
+ assert.ok((await s.get("U1", "group:room")).modelsByHarness.pi!.includes("gpt-5.6-terra"));
+ assert.ok(!(await s.get("U2", "group:room")).modelsByHarness.pi!.includes("gpt-5.6-terra"));
+ await s.built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "pi", modelId: "gpt-5.6-terra" });
+ s.built.config.setWebuiModels("org:default-org", ["gpt-5.6-sol"]);
+ await s.built.config.flushScope("org:default-org");
+ assert.ok(!(await s.get()).modelsByHarness.pi!.includes("gpt-5.6-terra"));
+ assert.equal((await s.put({ harnessId: "pi", modelId: "gpt-5.6-terra" })).status, 400);
+ const refused = await s.built.app.turn({
+ surface: "web",
+ actor: { externalId: "U1" },
+ conversation: { kind: "dm", threadRef: "web:U1:excluded-saved-model" },
+ text: "hello",
+ liveActor: true,
+ async: true,
+ harness: "pi",
+ model: "gpt-5.6-terra",
+ });
+ assert.equal(refused.status, "refused");
+ s.built.config.setWebuiModels("org:default-org", []);
+ await s.built.config.flushScope("org:default-org");
+ assert.ok((await s.get()).modelsByHarness.pi!.includes("gpt-5.6-terra"));
+ } finally {
+ await s.close();
+ }
+});