Multiplayer agent harness for work.
Fix model selection with personal AI credentials (#1556)
Joshua Francebe5bbf67db31
diff --git a/plugins/web-ui/src/composer.ts b/plugins/web-ui/src/composer.tsindex e4b4d93bc..465001555 100644--- a/plugins/web-ui/src/composer.ts+++ b/plugins/web-ui/src/composer.ts@@ -174,7 +174,10 @@ export function resyncModelSelection(): void {}export function createComposerSurface(ctx: ConvCtx): ComposerSurface {- const refreshAccount = () => ctx.chat.drawActiveChat();+ const refreshAccount = () => {+ loadoutRestored = false;+ void refreshRuntimeSelection(ctx.chat.state.scopeId, ctx.chat.state.agent ?? undefined, true);+ };window.addEventListener("model-account-changed", refreshAccount);let runtimeRequest = 0;let runtimeIdentity = "";@@ -485,7 +488,6 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {}const compact = Boolean(ctx.pane) || isPhone();- const showRuntimeControls = !appState.me?.individualModelAuth;const runtimeControls = modelPicker.render(agent, selectedModel, inputBlocked);return html`<form@@ -617,7 +619,7 @@ export function createComposerSurface(ctx: ConvCtx): ComposerSurface {>${icon(Paperclip, 18)}</button>- ${showRuntimeControls ? runtimeControls : nothing}+ ${runtimeControls}</div><div class="composer-right">${sendControls(agent)}</div></div>diff --git a/plugins/web-ui/src/runtime-config-store.ts b/plugins/web-ui/src/runtime-config-store.tsindex 52c234590..f19ad9e13 100644--- a/plugins/web-ui/src/runtime-config-store.ts+++ b/plugins/web-ui/src/runtime-config-store.ts@@ -14,6 +14,19 @@ type Entry = {const FRESH_MS = 30_000;const entries = new Map<string, Entry>();let bootScope: string | null = null;+let accountRevision = 0;++export function invalidateRuntimeConfigs(): void {+ accountRevision++;+ for (const entry of entries.values()) {+ entry.generation++;+ entry.config = null;+ entry.fetchedAt = 0;+ entry.load = null;+ }+}++if (typeof window !== "undefined") window.addEventListener("model-account-changed", invalidateRuntimeConfigs);function entryFor(scopeId: string): Entry {let entry = entries.get(scopeId);@@ -87,9 +100,10 @@ export async function saveRuntimeConfig(scopeId: string, change: Change): Promis// Invalidate old reads at enqueue time, including when the save ultimately fails.++entry.generation;entry.load = null;+ const revision = accountRevision;const save = (entry.write ?? Promise.resolve()).then(async () => {const config = await updateRuntimeConfig(scopeId, change);- publish(scopeId, entry, config);+ if (revision === accountRevision) publish(scopeId, entry, config);return config;});const settled = save.then(diff --git a/plugins/web-ui/test/composer-picker.test.ts b/plugins/web-ui/test/composer-picker.test.tsindex b403efebd..059c556f1 100644--- a/plugins/web-ui/test/composer-picker.test.ts+++ b/plugins/web-ui/test/composer-picker.test.ts@@ -25,7 +25,7 @@ function model(id: string, label: string, provider = "anthropic"): ModelMetadata};}-test("the shared picker preserves composer choices and saves context defaults", async () => {+test("the personal-account picker preserves composer choices and saves context defaults", async () => {const dom = new JSDOM('<!doctype html><div id="app"></div><div id="composer"></div>', {url: "http://localhost/web-ui/",pretendToBeVisual: true,@@ -48,6 +48,7 @@ test("the shared picker preserves composer choices and saves context defaults",};const updates: Record<string, unknown>[] = [];let failNextGet = true;+ let runtimeReads = 0;let failNextPut = false;let deferNextPut = false;let pendingPut: Promise<void> | undefined;@@ -78,6 +79,7 @@ test("the shared picker preserves composer choices and saves context defaults",},fetch: async (input: RequestInfo | URL, init?: RequestInit) => {if (String(input).startsWith("/api/runtime-config") && init?.method !== "PUT") {+ runtimeReads++;if (failNextGet) {failNextGet = false;return Response.json({ error: "initial load failed" }, { status: 500 });@@ -160,7 +162,7 @@ test("the shared picker preserves composer choices and saves context defaults",const { appState } = await vite.ssrLoadModule("/src/shell-state.ts");const { createComposerSurface } = await vite.ssrLoadModule("/src/composer.ts");const { render } = await vite.ssrLoadModule("lit");- appState.me = { user: "tester", org: "test" };+ appState.me = { user: "tester", org: "test", individualModelAuth: true, modelAuthConnected: true };const host = document.querySelector<HTMLElement>("#composer")!;const agent = { state: { isStreaming: false, messages: [] } } as unknown as Agent;const draw = (): void => render(composer!.composerForm(agent), host);@@ -217,6 +219,14 @@ test("the shared picker preserves composer choices and saves context defaults",await composer!.refreshRuntimeSelection(null, agent, true);assert.equal(composer!.state.effortLevel, "high", "identical refresh preserves restored effort");assert.equal(composer!.state.fastMode, true, "identical refresh preserves restored Fast");+ const readsBeforeAccountChange = runtimeReads;+ appState.me.individualModelAuth = false;+ window.dispatchEvent(new dom.window.CustomEvent("model-account-changed"));+ await tick();+ assert.equal(runtimeReads, readsBeforeAccountChange + 1);+ assert.ok(host.querySelector(".loadout-button"));+ appState.me.individualModelAuth = true;+ draw();const siblingHost = document.createElement("section");document.body.append(siblingHost);const siblingAgent = { state: { isStreaming: false, messages: [] } } as unknown as Agent;diff --git a/plugins/web-ui/test/composer-source.test.ts b/plugins/web-ui/test/composer-source.test.tsindex 58b1247b8..42d31cd6b 100644--- a/plugins/web-ui/test/composer-source.test.ts+++ b/plugins/web-ui/test/composer-source.test.ts@@ -24,7 +24,7 @@ test("compact and full composers share one left-side picker with Fast inside itsconst leftStart = composer.indexOf('class="composer-left"');const rightStart = composer.indexOf('class="composer-right"');assert.ok(leftStart >= 0 && rightStart > leftStart);- assert.ok(composer.slice(leftStart, rightStart).includes("showRuntimeControls ? runtimeControls : nothing"));+ assert.ok(composer.slice(leftStart, rightStart).includes("${runtimeControls}"));assert.ok(/class="composer-right">\$\{sendControls\(agent\)\}<\/div>/.test(composer));const loadout = picker.slice(picker.indexOf("function render"), picker.indexOf("function menuArrowKeys"));assert.ok(/role="menuitemcheckbox"\s+aria-label="Fast"/.test(loadout));diff --git a/plugins/web-ui/test/runtime-config-sync.test.ts b/plugins/web-ui/test/runtime-config-sync.test.tsindex 6e0bf4841..f1ac7bd0e 100644--- a/plugins/web-ui/test/runtime-config-sync.test.ts+++ b/plugins/web-ui/test/runtime-config-sync.test.ts@@ -2,6 +2,7 @@ import assert from "node:assert/strict";import { afterEach, test } from "node:test";import {getRuntimeConfig,+ invalidateRuntimeConfigs,loadRuntimeConfig,saveRuntimeConfig,seedRuntimeConfig,@@ -176,3 +177,37 @@ test("cached configuration is revalidated when returning to a view after the freDate.now = now;}});++test("account changes invalidate all scopes and prevent stale requests from republishing", async () => {+ const scope = "scope:account-race";+ const other = "scope:account-other";+ const stale = deferred<Response>();+ const old = runtimeConfig(scope);+ seedRuntimeConfig(other, runtimeConfig(other));+ let reads = 0;+ globalThis.fetch = async () => {+ reads++;+ return stale.promise;+ };+ const pending = loadRuntimeConfig(scope);+ invalidateRuntimeConfigs();+ assert.equal(getRuntimeConfig(other), null);+ const personal = runtimeConfig(scope, { effective: { harnessId: "pi", modelId: "personal" } });+ globalThis.fetch = async () => {+ reads++;+ return Response.json(personal);+ };+ await loadRuntimeConfig(scope, true);+ stale.resolve(Response.json(old));+ await pending;+ assert.equal(reads, 2);+ assert.deepEqual(getRuntimeConfig(scope), personal);++ const saving = deferred<Response>();+ globalThis.fetch = async () => saving.promise;+ const write = saveRuntimeConfig(scope, { modelId: "old-account" });+ invalidateRuntimeConfigs();+ saving.resolve(Response.json(old));+ await write;+ assert.equal(getRuntimeConfig(scope), null);+});diff --git a/src/api/app-turn.ts b/src/api/app-turn.tsindex 88c7f6ca8..f2bd12c59 100644--- a/src/api/app-turn.ts+++ b/src/api/app-turn.ts@@ -1,3 +1,4 @@+import { userRuntimeConfigBody } from "./runtime-config.ts";import { isSubagentThreadRef } from "../sessions/session-syscalls.ts";import type { Conversation, Principal, TurnRequest, TurnResult } from "../types.ts";import { orgId as orgIdOf } from "../config.ts";@@ -176,6 +177,8 @@ export function createTurnMethods(? await deps.config.getModelAccountDurable(actor.id): "company";const individualAuth = modelAccount !== "company";+ let requestedModel = req.model;+ let requestedHarness = req.harness;if (req.surface === "web") {const threadRef = req.conversation.threadRef;const existing = await deps.sessions.getByThread(threadRef);@@ -194,6 +197,20 @@ export function createTurnMethods(harnessId: fallbackHarness,modelId: defaultModelForHarness(fallbackHarness),};+ if (individualAuth && (req.model || req.harness)) {+ const available = await userRuntimeConfigBody({ deps }, targetScope, actor.id);+ const harness = req.harness ?? available.effective.harnessId;+ const model = req.model ?? available.effective.modelId;+ if (!available.modelsByHarness[harness]?.includes(model))+ return { status: "refused", reason: "Your connected AI account cannot serve this model on that harness." };+ const invalidModelOption = validateWebTurnModelOptions(+ { ...req, model },+ available.modelsByHarness[harness] ?? [],+ );+ if (invalidModelOption) return { status: "refused", reason: invalidModelOption };+ requestedModel = model;+ requestedHarness = harness;+ }if (!individualAuth) {const configuredKeys = deps.providerKeys ??deps.modelProviders ?? { anthropic: false, openai: false, openrouter: false };@@ -304,8 +321,8 @@ export function createTurnMethods(...(req.detectOpener ? { detectOpener: req.detectOpener } : {}),...(req.attachments?.length ? { attachments: req.attachments } : {}),...(req.inboundNotes?.length ? { inboundNotes: req.inboundNotes } : {}),- ...(!individualAuth && req.harness ? { harness: req.harness } : {}),- ...(!individualAuth && req.model ? { model: req.model } : {}),+ ...((!individualAuth || req.surface === "web") && requestedHarness ? { harness: requestedHarness } : {}),+ ...((!individualAuth || req.surface === "web") && requestedModel ? { model: requestedModel } : {}),...turnModelOptions(req),...(req.readOnly ? { readOnly: true } : {}),...(privateRequestdiff --git a/src/api/routes/surface.ts b/src/api/routes/surface.tsindex 8c9901dc8..0e6518d62 100644--- a/src/api/routes/surface.ts+++ b/src/api/routes/surface.ts@@ -1,6 +1,6 @@import { isSessionStatus } from "../../sessions/session-status.ts";import { suggestedActivityRoutes } from "./suggested-activities.ts";-import { runtimeFallback, runtimeConfigBody, webuiModelEnabled } from "../runtime-config.ts";+import { runtimeFallback, userRuntimeConfigBody, webuiModelEnabled } from "../runtime-config.ts";import { sessionSharingRoutes } from "./session-sharing.ts";import type { Grant, ScopeId, Session } from "../../types.ts";import { parseScopeId, scopeId as makeScopeId } from "../../types.ts";@@ -1218,7 +1218,7 @@ async function getRuntimeConfig(ctx: ApiCtx): Promise<void> {const target = await runtimeTarget(ctx);if (!target) return sendJson(ctx.res, 403, { error: "forbidden" });await ctx.deps.refreshModels?.();- return sendJson(ctx.res, 200, await runtimeConfigBody(ctx, target.scope));+ return sendJson(ctx.res, 200, await userRuntimeConfigBody(ctx, target.scope, target.actorId));}async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {@@ -1262,6 +1262,11 @@ async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {const fastMode = ctx.body.fastMode ?? false;if (typeof fastMode !== "boolean") return sendJson(ctx.res, 400, { error: "fast_mode_invalid" });const choice = { harnessId, modelId, effortLevel, fastMode: fastMode && fastModeModelIds().includes(modelId) };+ if ((await config.getModelAccountDurable(target.actorId)) !== "company") {+ const available = await userRuntimeConfigBody(ctx, target.scope, target.actorId);+ if (!available.modelsByHarness[harnessId]?.includes(modelId))+ return sendJson(ctx.res, 400, { error: "account_runtime_unavailable" });+ }await config.setRuntimeSelectionLatest(target.scope, choice);}audit(ctx.deps, {@@ -1270,7 +1275,7 @@ async function putRuntimeConfig(ctx: ApiCtx): Promise<void> {resource: "runtime-config",scopeLabel: target.scope,});- return sendJson(ctx.res, 200, await runtimeConfigBody(ctx, target.scope));+ return sendJson(ctx.res, 200, await userRuntimeConfigBody(ctx, target.scope, target.actorId));}async function getChannelHeaderPin(ctx: ApiCtx): Promise<void> {diff --git a/src/api/runtime-config.ts b/src/api/runtime-config.tsindex 3068f47ba..e50641075 100644--- a/src/api/runtime-config.ts+++ b/src/api/runtime-config.ts@@ -1,7 +1,8 @@+import { resolveIndividualAuthRouting } from "../core/individual-auth-routing.ts";import { gatewayModelCatalog } from "../model/gateway-models.ts";-import type { ServerDeps } from "./deps.ts";+import type { AppDeps } from "./app-types.ts";import type { ScopeId } from "../types.ts";-import { orgScope } from "./routes/shared.ts";+import { orgScope } from "../config.ts";import {defaultModelForHarness,isHarnessId,@@ -16,21 +17,24 @@ import {thinkingLevelsForHarness,harnessSupportsFastMode,codexProviderModelId,+ codexSubscriptionModelId,type HarnessId,} from "../model/pi-models.ts";import { builtInModelCatalog, selectableCatalogForHarness, selectableModelCatalog } from "../model/model-catalog.ts";import type { RuntimeChoice } from "../harness/harness.ts";-export type RuntimeDeps = Pick<- ServerDeps,- | "config"- | "harnessId"- | "baseModelDefault"- | "providerKeys"- | "modelCredentials"- | "modelCredentialFetch"- | "refreshModels"->;+export type RuntimeDeps = Partial<+ Pick<+ AppDeps,+ | "config"+ | "harnessId"+ | "providerKeys"+ | "modelCredentials"+ | "modelCredentialFetch"+ | "refreshModels"+ | "userModelCredentials"+ >+> & { baseModelDefault?: string };export function runtimeFallback(ctx: { deps: RuntimeDeps }): { harnessId: HarnessId; modelId: string } {const harnessId = isHarnessId(ctx.deps.harnessId) ? ctx.deps.harnessId : "pi";@@ -42,12 +46,55 @@ export function runtimeFallback(ctx: { deps: RuntimeDeps }): { harnessId: Harnesreturn { harnessId, modelId: ctx.deps.baseModelDefault ?? defaultModelForHarness(harnessId, undefined, providers) };}-export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: ScopeId) {+export async function userRuntimeConfigBody(ctx: { deps: RuntimeDeps }, scope: ScopeId, actorId: string) {+ const account = await ctx.deps.config?.getModelAccountDurable(actorId);+ const store = ctx.deps.userModelCredentials;+ if (!store || !account || account === "company") return runtimeConfigBody(ctx, scope);+ const [anthropic, openai] = await Promise.all([+ account === "openai" ? null : store.get(actorId, "anthropic"),+ account === "anthropic" ? null : store.get(actorId, "openai"),+ ]);+ const snapshot = await runtimeConfigBody(ctx, scope, async (choice) => {+ const route = resolveIndividualAuthRouting(anthropic, openai, choice.modelId, choice.harnessId);+ return route?.harness === choice.harnessId && route.model === choice.modelId ? null : "account_runtime_unavailable";+ });+ const route = resolveIndividualAuthRouting(+ anthropic,+ openai,+ snapshot.effective.modelId,+ snapshot.effective.harnessId,+ );+ if (!route?.model || !snapshot.modelsByHarness[route.harness]?.includes(route.model)) return snapshot;+ const { unavailableReason: _, ...available } = snapshot;+ return {+ ...available,+ effective: {+ ...snapshot.effective,+ harnessId: route.harness,+ modelId: route.model,+ effortLevel: thinkingLevelsForHarness(route.harness).includes(snapshot.effective.effortLevel ?? "auto")+ ? snapshot.effective.effortLevel+ : "auto",+ fastMode:+ snapshot.effective.fastMode === true &&+ harnessSupportsFastMode(route.harness) &&+ fastModeModelIds().includes(route.model),+ },+ };+}++export async function runtimeConfigBody(+ ctx: { deps: RuntimeDeps },+ scope: ScopeId,+ authorizeChoice?: (choice: RuntimeChoice) => Promise<string | null>,+) {+ if (authorizeChoice)+ ctx = { deps: { ...ctx.deps, providerKeys: ALL_PROVIDERS_AVAILABLE, modelCredentials: undefined } };const config = ctx.deps.config!;const configuredKeys = ctx.deps.providerKeys ?? ALL_PROVIDERS_AVAILABLE;const managedKeys = ctx.deps.modelCredentials ? await ctx.deps.modelCredentials.availability() : configuredKeys;const fallback = runtimeFallback(ctx);- const org = orgScope(ctx.deps);+ const org = orgScope();const approvedHarnesses = ((await config.getApprovedHarnessesDurable()) ?? [fallback.harnessId]).filter(isHarnessId);const providersFor = (harnessId: string) => modelProviderAvailabilityFor(harnessId, configuredKeys, managedKeys);const catalog =@@ -108,6 +155,10 @@ export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: Scopefor (const choice of selected) {if (allowlist?.length !== 0 &&+ (!authorizeChoice ||+ !allowlist ||+ choice.modelId === orgDefault.modelId ||+ allowlist.includes(codexProviderModelId(choice.modelId))) &&choice.harnessId === harnessId &&modelSupportedByHarness(choice.modelId, harnessId) &&!ids.includes(choice.modelId)@@ -117,6 +168,21 @@ export async function runtimeConfigBody(ctx: { deps: RuntimeDeps }, scope: Scopereturn [harnessId, serviceableModelIds(ids, providersFor(harnessId))];}),);+ if (authorizeChoice) {+ const piModels = modelsByHarness.pi ?? [];+ for (const id of piModels) {+ const subscriptionId = codexSubscriptionModelId(id);+ if (modelSupportedByHarness(subscriptionId, "pi") && !piModels.includes(subscriptionId))+ piModels.push(subscriptionId);+ }+ for (const harnessId of approvedHarnesses) {+ const candidates = modelsByHarness[harnessId] ?? [];+ const allowed = await Promise.all(+ candidates.map((modelId) => authorizeChoice({ harnessId, modelId, effortLevel: "auto", fastMode: false })),+ );+ modelsByHarness[harnessId] = candidates.filter((_, index) => !allowed[index]);+ }+ }const advertisedModelIds = new Set(Object.values(modelsByHarness).flat());const modelCatalog = Object.fromEntries([...advertisedModelIds].flatMap((id) => {@@ -162,10 +228,10 @@ export function validateRuntimeChoice(choice: RuntimeChoice): string | null {export async function webuiModelEnabled(ctx: { deps: RuntimeDeps }, modelId: string): Promise<boolean> {modelId = codexProviderModelId(modelId);const config = ctx.deps.config!;- const picker = await config.getWebuiModelsDurable(orgScope(ctx.deps));+ const picker = await config.getWebuiModelsDurable(orgScope());if (picker == null || picker.includes(modelId)) return true;if (picker.length === 0) return false;- const org = orgScope(ctx.deps);+ const org = orgScope();const stored = await config.getRuntimeSelectionDurable(org);const orgModel = stored?.modelId ?? (await config.getBaseModelOwnDurable(org)) ?? runtimeFallback(ctx).modelId;return modelId === orgModel;diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.tsindex c8e172824..09eac66fc 100644--- a/src/core/orchestrator.ts+++ b/src/core/orchestrator.ts@@ -2958,8 +2958,8 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {const routing = resolveIndividualAuthRouting(anthCred ?? null,oaiCred ?? null,- account === "personal" ? (runtime.modelId ?? input.model) : runtime.modelId,- account === "personal" ? preferredHarness : runtime.harnessId,+ account === "personal" || input.surface === "web" ? (runtime.modelId ?? input.model) : runtime.modelId,+ account === "personal" || input.surface === "web" ? preferredHarness : runtime.harnessId,);if (routing?.kind === "apikey") {userHarnessOverride = "pi";@@ -3007,6 +3007,13 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator {};let { userProviderKeys, userModelOverride, userHarnessOverride, claudeOauthToken, codexTurnAuth } =await loadRuntimeAuth({});+ if (+ input.surface === "web" &&+ userHarnessOverride &&+ ((input.model && input.model !== userModelOverride) ||+ (input.harness && input.harness !== userHarnessOverride))+ )+ throw new NonRetryableTurnError("Your connected AI account cannot serve this model on that harness.");const effectiveModel = userModelOverride ?? input.model;const effectiveHarness = userHarnessOverride ?? input.harness;if (userHarnessOverride) {diff --git a/src/harness/runtime-control.ts b/src/harness/runtime-control.tsindex d9001ef5c..2a2f32f43 100644--- a/src/harness/runtime-control.ts+++ b/src/harness/runtime-control.ts@@ -9,14 +9,7 @@ import {} from "../api/runtime-config.ts";import { livePersonCapability } from "../api/artifact-share.ts";import { parseScopeId } from "../types.ts";-import {- ALL_PROVIDERS_AVAILABLE,- isHarnessId,- thinkingLevelsForHarness,- safeModelMetadata,- modelSupportedByHarness,- codexSubscriptionModelId,-} from "../model/pi-models.ts";+import { isHarnessId, thinkingLevelsForHarness } from "../model/pi-models.ts";export function createRuntimeService(deps: RuntimeDeps, app: Pick<App, "authorizesCapabilityScope">): RuntimeService {return async (claims, active, request, authorizeChoice, individualAuth, signal) => {@@ -29,31 +22,7 @@ export function createRuntimeService(deps: RuntimeDeps, app: Pick<App, "authoriz)return { ok: false, error: "forbidden" };await deps.refreshModels?.();- const snapshot = await runtimeConfigBody(- { deps: individualAuth ? { ...deps, providerKeys: ALL_PROVIDERS_AVAILABLE, modelCredentials: undefined } : deps },- claims.scopeId,- );- if (individualAuth && authorizeChoice) {- if (snapshot.approvedHarnesses.includes("pi")) {- const piModels = snapshot.modelsByHarness.pi ?? [];- for (const id of piModels) {- const subscriptionId = codexSubscriptionModelId(id);- if (subscriptionId === id) continue;- if (modelSupportedByHarness(subscriptionId, "pi") && !piModels.includes(subscriptionId)) {- piModels.push(subscriptionId);- const metadata = safeModelMetadata(subscriptionId);- if (metadata) snapshot.modelCatalog[subscriptionId] = metadata;- }- }- }- for (const harnessId of snapshot.approvedHarnesses) {- const candidates = snapshot.modelsByHarness[harnessId] ?? [];- const allowed = await Promise.all(- candidates.map((modelId) => authorizeChoice({ harnessId, modelId, effortLevel: "auto", fastMode: false })),- );- snapshot.modelsByHarness[harnessId] = candidates.filter((_, index) => !allowed[index]);- }- }+ const snapshot = await runtimeConfigBody({ deps }, claims.scopeId, individualAuth ? authorizeChoice : undefined);if (request.action === "get")return {ok: true,diff --git a/test/personal-model-account-execution.test.ts b/test/personal-model-account-execution.test.tsindex 5ed91ac32..7fbd6bdc4 100644--- a/test/personal-model-account-execution.test.ts+++ b/test/personal-model-account-execution.test.ts@@ -82,3 +82,183 @@ test("disconnecting a queued personal account fails without invoking any main haawait built.runtime.stop();}});++for (const account of ["personal", "openai", "anthropic"] as const) {+ test(`web ${account} account honors the chosen model, effort and fast mode`, async () => {+ turns.length = 0;+ const provider = account === "anthropic" ? "anthropic" : "openai";+ const model = provider === "anthropic" ? "claude-sonnet-5" : "gpt-5.6-terra";+ const built = buildApp(testConfig({ openaiApiKey: "company-openai" }));+ built.config.setApprovedHarnesses(["pi"]);+ await built.config.flushScope("org:default-org");+ await built.userModelCredentials.setApiKey("U1", provider, `personal-${provider}`);+ await built.config.setPersonalModelAuth("U1", true, account === "personal" ? undefined : account);+ const submitted = await built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: `web:U1:picker-${account}` },+ text: "hello",+ liveActor: true,+ async: true,+ model,+ harness: "pi",+ thinkingLevel: "high",+ fastMode: true,+ });+ assert.ok(submitted.runId, JSON.stringify(submitted));+ const queued = await built.runs.get(submitted.runId!);+ assert.equal(queued?.request.model, model);+ assert.equal(queued?.request.harness, "pi");+ built.runtime.start();+ try {+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);+ assert.equal(finished.status, "done", JSON.stringify(finished.result));+ assert.equal(turns.length, 1);+ assert.deepEqual(turns[0]!.providerKeys, { [provider]: `personal-${provider}` });+ assert.deepEqual(turns[0]!.runtime, {+ harnessId: "pi",+ modelId: model,+ effortLevel: "high",+ fastMode: true,+ });+ } finally {+ await built.runtime.stop();+ }+ });+}++test("personal web selections reject wrong providers, harnesses, policy exclusions and disconnected keys", async () => {+ const built = buildApp(testConfig({ openaiApiKey: "company-openai", anthropicApiKey: "company-anthropic" }));+ built.config.setApprovedHarnesses(["pi", "claude"]);+ await built.config.flushScope("org:default-org");+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");+ await built.config.setPersonalModelAuth("U1", true, "openai");+ const submit = (model: string, harness = "pi") =>+ built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: `web:U1:invalid-${crypto.randomUUID()}` },+ text: "hello",+ liveActor: true,+ async: true,+ model,+ harness,+ });+ for (const [model, harness] of [+ ["claude-sonnet-5", "pi"],+ ["gpt-5.6-terra", "claude"],+ ["unknown", "pi"],+ ])+ assert.equal((await submit(model!, harness!)).status, "refused");+ built.config.setWebuiModels("org:default-org", ["gpt-5.6-sol"]);+ await built.config.flushScope("org:default-org");+ assert.equal((await submit("gpt-5.6-terra")).status, "refused");+ await built.userModelCredentials.delete("U1", "openai");+ assert.equal((await submit("gpt-5.6-sol")).status, "refused");+});++test("a queued web selection cannot fall back to another personal provider after disconnect", async () => {+ turns.length = 0;+ const built = buildApp(testConfig());+ built.config.setApprovedHarnesses(["pi"]);+ await built.config.flushScope("org:default-org");+ await built.userModelCredentials.setApiKey("U1", "anthropic", "personal-anthropic");+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");+ await built.config.setPersonalModelAuth("U1", true);+ const submitted = await built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "web:U1:disconnect-after-pick" },+ text: "hello",+ liveActor: true,+ async: true,+ model: "gpt-5.6-terra",+ harness: "pi",+ });+ assert.ok(submitted.runId, JSON.stringify(submitted));+ await built.userModelCredentials.delete("U1", "openai");+ built.runtime.start();+ try {+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);+ assert.equal(finished.status, "failed");+ assert.match(JSON.stringify(finished), /cannot serve this model/);+ assert.equal(turns.length, 0);+ } finally {+ await built.runtime.stop();+ }+});++test("web subscription selections use the namespaced model and personal OAuth on Pi", async () => {+ turns.length = 0;+ const built = buildApp(testConfig());+ built.config.setApprovedHarnesses(["pi"]);+ await built.config.flushScope("org:default-org");+ await built.userModelCredentials.setOAuth("U1", "openai", {+ accessToken: "personal-oauth",+ expiresAt: Date.now() + 3_600_000,+ });+ await built.config.setPersonalModelAuth("U1", true, "openai");+ const submitted = await built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "web:U1:oauth-picker" },+ text: "hello",+ liveActor: true,+ async: true,+ model: "codex/gpt-5.6-terra",+ harness: "pi",+ thinkingLevel: "high",+ fastMode: false,+ });+ assert.ok(submitted.runId, JSON.stringify(submitted));+ built.runtime.start();+ try {+ const finished = await built.runs.waitFor(submitted.runId!, 5_000);+ assert.equal(finished.status, "done", JSON.stringify(finished.result));+ assert.equal(turns.length, 1);+ assert.deepEqual(turns[0]!.providerKeys, { "openai-codex": "personal-oauth" });+ assert.deepEqual(turns[0]!.runtime, {+ harnessId: "pi",+ modelId: "codex/gpt-5.6-terra",+ effortLevel: "high",+ fastMode: false,+ });+ } finally {+ await built.runtime.stop();+ }+});++test("partial personal web choices queue the complete validated scoped runtime", async () => {+ const built = buildApp(testConfig());+ built.config.setApprovedHarnesses(["pi", "codex"]);+ await built.config.setRuntimeSelectionLatest("org:default-org", { harnessId: "pi", modelId: "gpt-5.6-sol" });+ await built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "pi", modelId: "gpt-5.6-terra" });+ await built.userModelCredentials.setApiKey("U1", "openai", "personal-openai");+ await built.config.setPersonalModelAuth("U1", true, "openai");+ const submit = (choice: { harness?: string; model?: string }) =>+ built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: `web:U1:partial-${crypto.randomUUID()}` },+ text: "hello",+ liveActor: true,+ async: true,+ ...choice,+ });+ const first = await submit({ harness: "pi" });+ assert.ok(first.runId, JSON.stringify(first));+ const queued = await built.runs.get(first.runId!);+ assert.equal(queued?.request.harness, "pi");+ assert.equal(queued?.request.model, "gpt-5.6-terra");+ await built.userModelCredentials.delete("U1", "openai");+ await built.userModelCredentials.setOAuth("U1", "openai", {+ accessToken: "personal-oauth",+ expiresAt: Date.now() + 3_600_000,+ });+ await built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "codex", modelId: "gpt-5.6-sol" });+ const second = await submit({ model: "gpt-5.6-terra" });+ assert.ok(second.runId, JSON.stringify(second));+ const oauth = await built.runs.get(second.runId!);+ assert.equal(oauth?.request.harness, "codex");+ assert.equal(oauth?.request.model, "gpt-5.6-terra");+});diff --git a/test/personal-runtime-picker.test.ts b/test/personal-runtime-picker.test.tsnew file mode 100644index 000000000..289bffad3--- /dev/null+++ b/test/personal-runtime-picker.test.ts@@ -0,0 +1,129 @@+import "./support/auto-fake-sprites.ts";+import assert from "node:assert/strict";+import { test } from "node:test";+import type { AddressInfo } from "node:net";+import { buildApp } from "../src/wiring.ts";+import { createInsecureTestServer } from "../src/api/server.ts";+import { testConfig } from "./support/test-config.ts";+import type { userRuntimeConfigBody } from "../src/api/runtime-config.ts";+import { resolveModel } from "../src/model/pi-models.ts";++type Snapshot = Awaited<ReturnType<typeof userRuntimeConfigBody>>;++async function setup() {+ const built = buildApp(testConfig());+ built.config.setApprovedHarnesses(["pi", "claude", "codex"]);+ await built.config.flushScope("org:default-org");+ const server = createInsecureTestServer(built.app, {+ config: built.config,+ userModelCredentials: built.userModelCredentials,+ harnessId: "pi",+ providerKeys: { anthropic: false, openai: false, openrouter: false },+ });+ server.listen(0);+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;+ const get = async (user = "U1", scope = `personal:${user}`) => {+ const response = await fetch(`${base}/v1/runtime-config?principalId=${user}&scopeId=${encodeURIComponent(scope)}`);+ assert.equal(response.status, 200);+ return response.json() as Promise<Snapshot>;+ };+ const put = (choice: object) =>+ fetch(`${base}/v1/runtime-config`, {+ method: "PUT",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({ principalId: "U1", scopeId: "personal:U1", ...choice }),+ });+ return { built, get, put, close: () => new Promise<void>((r) => server.close(() => r())) };+}++test("personal API-key picker works without company keys and saves model, effort and Fast", async () => {+ const s = await setup();+ try {+ await s.built.userModelCredentials.setApiKey("U1", "openai", "synthetic-openai");+ await s.built.config.setPersonalModelAuth("U1", true, "openai");+ const before = await s.get();+ assert.ok(before.modelsByHarness.pi!.includes("gpt-5.6-terra"));+ assert.ok(before.modelsByHarness.pi!.every((id: string) => resolveModel(id)?.provider === "openai"));+ assert.deepEqual(before.modelsByHarness.claude, []);+ assert.deepEqual(before.modelsByHarness.codex, []);+ assert.ok(before.modelsByHarness.pi!.includes(before.effective.modelId));+ assert.equal(before.unavailableReason, undefined);+ assert.doesNotMatch(JSON.stringify(before), /synthetic-openai/);+ const choice = { harnessId: "pi", modelId: "gpt-5.6-terra", effortLevel: "high", fastMode: true };+ const saved = await s.put(choice);+ assert.equal(saved.status, 200);+ assert.deepEqual(((await saved.json()) as Snapshot).effective, choice);+ assert.deepEqual((await s.get()).effective, choice);+ assert.equal((await s.put({ ...choice, modelId: "claude-sonnet-5" })).status, 400);+ await s.built.userModelCredentials.delete("U1", "openai");+ assert.deepEqual((await s.get()).modelsByHarness.pi, []);+ assert.equal((await s.put(choice)).status, 400);+ } finally {+ await s.close();+ }+});++for (const provider of ["anthropic", "openai"] as const) {+ test(`personal ${provider} OAuth picker advertises only compatible runtimes`, async () => {+ const s = await setup();+ try {+ await s.built.userModelCredentials.setOAuth("U1", provider, {+ accessToken: "synthetic-access",+ refreshToken: "synthetic-refresh",+ expiresAt: Date.now() + 3_600_000,+ });+ await s.built.config.setPersonalModelAuth("U1", true, provider);+ const config = await s.get();+ if (provider === "anthropic") {+ assert.deepEqual(config.modelsByHarness.pi, []);+ assert.ok(config.modelsByHarness.claude!.includes("claude-sonnet-5"));+ assert.deepEqual(config.modelsByHarness.codex, []);+ } else {+ assert.ok(config.modelsByHarness.pi!.includes("codex/gpt-5.6-sol"));+ assert.ok(!config.modelsByHarness.pi!.includes("gpt-5.6-sol"));+ assert.ok(config.modelsByHarness.codex!.includes("gpt-5.6-sol"));+ assert.deepEqual(config.modelsByHarness.claude, []);+ }+ assert.ok(config.modelsByHarness[config.effective.harnessId]!.includes(config.effective.modelId));+ } finally {+ await s.close();+ }+ });+}++test("personal picker keeps org restrictions and shared-scope caller isolation", async () => {+ const s = await setup();+ try {+ await s.built.userModelCredentials.setApiKey("U1", "openai", "synthetic-openai");+ await s.built.userModelCredentials.setApiKey("U2", "anthropic", "synthetic-anthropic");+ await s.built.config.setPersonalModelAuth("U1", true, "openai");+ await s.built.config.setPersonalModelAuth("U2", true, "anthropic");+ await s.built.directory.replaceGroups([+ { groupId: "room", principalId: "U1" },+ { groupId: "room", principalId: "U2" },+ ]);+ assert.ok((await s.get("U1", "group:room")).modelsByHarness.pi!.includes("gpt-5.6-terra"));+ assert.ok(!(await s.get("U2", "group:room")).modelsByHarness.pi!.includes("gpt-5.6-terra"));+ await s.built.config.setRuntimeSelectionLatest("personal:U1", { harnessId: "pi", modelId: "gpt-5.6-terra" });+ s.built.config.setWebuiModels("org:default-org", ["gpt-5.6-sol"]);+ await s.built.config.flushScope("org:default-org");+ assert.ok(!(await s.get()).modelsByHarness.pi!.includes("gpt-5.6-terra"));+ assert.equal((await s.put({ harnessId: "pi", modelId: "gpt-5.6-terra" })).status, 400);+ const refused = await s.built.app.turn({+ surface: "web",+ actor: { externalId: "U1" },+ conversation: { kind: "dm", threadRef: "web:U1:excluded-saved-model" },+ text: "hello",+ liveActor: true,+ async: true,+ harness: "pi",+ model: "gpt-5.6-terra",+ });+ assert.equal(refused.status, "refused");+ s.built.config.setWebuiModels("org:default-org", []);+ await s.built.config.flushScope("org:default-org");+ assert.ok((await s.get()).modelsByHarness.pi!.includes("gpt-5.6-terra"));+ } finally {+ await s.close();+ }+});