Multiplayer agent harness for work.
Let custom providers serve models QM already ships, under their own name (#2427)
Joshua France9eda1297e2b9
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 79396c662..6b5257267 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -75,7 +75,7 @@ import {codexProviderModelId,modelHasPrices,} from "../model/pi-models.ts";-import { customModelsJson, customProvidersVersion } from "../model/custom-providers.ts";+import { customModelsJson, customProvidersVersion, resolveCustomModel } from "../model/custom-providers.ts";import {GatewayModelUnavailableError,modelGatewayRequest,@@ -1204,7 +1204,7 @@ export async function buildModelRuntime(const providerModelId =model.provider === CODEX_SUBSCRIPTION_PROVIDER? codexProviderModelId(model.id)- : (modelRequestOverrides(model.id)?.model ?? model.id);+ : (modelRequestOverrides(model.id)?.model ?? resolveCustomModel(model.id)?.upstreamId ?? model.id);const candidate = withRequestHeaders(model, true, false);const passthrough = {...retained(options),diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.tsindex 442cdb84e..858d7ed2b 100644--- a/src/model/custom-providers.ts+++ b/src/model/custom-providers.ts@@ -13,7 +13,6 @@* resolved per-call by wiring alongside the built-in provider keys.*/-import { modelIdReserved } from "./pi-models.ts";import { parseProviderBaseUrl, PROVIDER_IDS } from "./provider-endpoints.ts";export const CUSTOM_PROVIDER_PROTOCOLS = ["openai", "openai-responses", "anthropic"] as const;@@ -65,8 +64,6 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {throw new Error(`model "${m.id}": name must be a string of 200 chars or fewer`);if (m.reasoning !== undefined && typeof m.reasoning !== "boolean")throw new Error(`model "${m.id}": reasoning must be a boolean`);- if (modelIdReserved(m.id) || (registry.has(m.id) && registry.get(m.id)?.provider !== spec.id))- throw new Error(`model id "${m.id}" is already registered`);if (seen.has(m.id)) throw new Error(`duplicate model id "${m.id}"`);seen.add(m.id);for (const [field, v] of [@@ -95,6 +92,7 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {*/export interface CustomRuntimeModel {id: string;+ upstreamId: string;name: string;provider: string;api: "openai-completions" | "openai-responses" | "anthropic-messages";@@ -117,7 +115,8 @@ function customProviderApi(protocol: CustomProviderProtocol): CustomRuntimeModelfunction toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): CustomRuntimeModel {return {- id: m.id,+ id: `custom/${provider.id}/${m.id}`,+ upstreamId: m.id,name: m.name?.trim() || m.id,provider: provider.id,api: customProviderApi(provider.protocol),@@ -146,7 +145,8 @@ export function setCustomProviders(specs: CustomProviderSpec[]): void {const next = new Map<string, CustomRuntimeModel>();for (const spec of specs) {for (const m of spec.models) {- next.set(m.id, toRuntimeModel(spec, m));+ const model = toRuntimeModel(spec, m);+ next.set(model.id, model);}}registry = next;@@ -188,14 +188,16 @@ export function customModelsJson(): { providers: Record<string, unknown> } | undname: spec.name,baseUrl: spec.baseUrl,api: customProviderApi(spec.protocol),- models: spec.models.map((m) => ({- id: m.id,- name: m.name ?? m.id,- reasoning: m.reasoning ?? false,- contextWindow: m.contextWindow ?? 128_000,- maxTokens: m.maxTokens ?? 8_192,- cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },- })),+ models: [...registry.values()]+ .filter((m) => m.provider === spec.id)+ .map(({ id, name, reasoning, contextWindow, maxTokens, cost }) => ({+ id,+ name,+ reasoning,+ contextWindow,+ maxTokens,+ cost,+ })),},]),),diff --git a/src/model/pi-models.ts b/src/model/pi-models.tsindex 7ea212cf3..8100003a4 100644--- a/src/model/pi-models.ts+++ b/src/model/pi-models.ts@@ -415,12 +415,7 @@ export function modelIdReserved(id: string): boolean {export function validateModelOverlay(value: unknown): RegisteredModel {const spec = parseRegisteredModel(value);- if (- REGISTRY_BY_ID.has(spec.id) ||- builtinModel(spec.id) ||- isCustomModelId(spec.id) ||- OPENROUTER_CATALOG_MODELS.has(spec.id)- )+ if (REGISTRY_BY_ID.has(spec.id) || builtinModel(spec.id) || OPENROUTER_CATALOG_MODELS.has(spec.id))throw new Error("model id is already registered");if (isEmbeddingModel(spec)) return spec;const template = resolveBuiltinModel(spec.template);@@ -450,7 +445,7 @@ export function setModelOverlays(}if (isEmbeddingModel(spec)) {if (verificationFailures.has(id)) unavailable.set(id, verificationFailures.get(id)!);- else if (!isCustomModelId(id)) next.set(id, spec);+ else next.set(id, spec);continue;}const template = resolveBuiltinModel(spec.template);@@ -459,7 +454,7 @@ export function setModelOverlays(id,"Builtin template is unavailable or has a different provider; ask an administrator to repair this model",);- } else if (isCustomModelId(id) || OPENROUTER_CATALOG_MODELS.has(id)) {+ } else if (OPENROUTER_CATALOG_MODELS.has(id)) {unavailable.set(id, "Model id conflicts with another provider; ask an administrator to repair this model");} else if (verificationFailures.has(id)) unavailable.set(id, verificationFailures.get(id)!);else next.set(id, spec);diff --git a/test/custom-provider-boot-wiring.test.ts b/test/custom-provider-boot-wiring.test.tsindex 3c7730c40..1bf21e821 100644--- a/test/custom-provider-boot-wiring.test.ts+++ b/test/custom-provider-boot-wiring.test.ts@@ -20,19 +20,19 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defaconst config = testConfig({dataDir: mkdtempSync(join(tmpdir(), "custom-provider-boot-")),harness: "pi",- modelId: "acme-large",+ modelId: "custom/acme-gateway/acme-large",});const built = buildApp(config, { modelCredentialFetch: async () => new Response(null, { status: 200 }) });const deps = serverDeps(config, built);assert.equal(deps.customProviders, built.customProviders);assert.equal(deps.refreshCustomProviders, built.refreshCustomProviders);- assert.equal(deps.baseModelDefault, "acme-large");+ assert.equal(deps.baseModelDefault, "custom/acme-gateway/acme-large");const server = createInsecureTestServer(built.app, deps);server.listen(0);const base = `http://localhost:${(server.address() as AddressInfo).port}`;try {- assert.notEqual(defaultModelForHarness("pi", deps.baseModelDefault), "acme-large");+ assert.notEqual(defaultModelForHarness("pi", deps.baseModelDefault), "custom/acme-gateway/acme-large");const list = await fetch(`${base}/v1/admin/custom-providers`, { headers: ADMIN });assert.equal(list.status, 200);@@ -51,7 +51,7 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defa});assert.equal(put.status, 200);- assert.equal(defaultModelForHarness("pi", deps.baseModelDefault), "acme-large");+ assert.equal(defaultModelForHarness("pi", deps.baseModelDefault), "custom/acme-gateway/acme-large");const runtime = await fetch(`${base}/v1/runtime-config?principalId=admin-alice@default-org&scopeId=personal:admin-alice@default-org`,@@ -63,9 +63,9 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defamodelsByHarness: Record<string, string[]>;modelCatalog: Record<string, { name: string; provider: string }>;};- assert.equal(body.effective.modelId, "acme-large");- assert.ok(body.modelsByHarness.pi?.includes("acme-large"));- assert.equal(body.modelCatalog["acme-large"]?.provider, "acme-gateway");+ assert.equal(body.effective.modelId, "custom/acme-gateway/acme-large");+ assert.ok(body.modelsByHarness.pi?.includes("custom/acme-gateway/acme-large"));+ assert.equal(body.modelCatalog["custom/acme-gateway/acme-large"]?.provider, "acme-gateway");} finally {await new Promise<void>((resolve) => server.close(() => resolve()));}diff --git a/test/custom-provider-e2e.test.ts b/test/custom-provider-e2e.test.tsindex bb3c8b757..bf54ad4cd 100644--- a/test/custom-provider-e2e.test.ts+++ b/test/custom-provider-e2e.test.ts@@ -144,13 +144,13 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()assert.ok(!listing.includes("sk-qa-good"), "key never readable");// 5. model resolves like a built-in and is catalog-visible- const model = resolveModel("qa-chat");+ const model = resolveModel("custom/qa/qa-chat");assert.ok(model, "custom model resolves");assert.equal(model!.provider, "qa");assert.equal(model!.reasoning, true);assert.equal((model as { baseUrl?: string }).baseUrl, upstreamUrl);- assert.equal(modelSupportedByHarness("qa-chat", "pi"), true);- assert.equal(modelServiceable("qa-chat", { anthropic: false, openai: false, openrouter: false }), true);+ assert.equal(modelSupportedByHarness("custom/qa/qa-chat", "pi"), true);+ assert.equal(modelServiceable("custom/qa/qa-chat", { anthropic: false, openai: false, openrouter: false }), true);// 6. REAL model call through QM's pi path → fake upstream answersconst reply = await oneShot(@@ -183,7 +183,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()// 8. delete: models leave the registryr = await api("/v1/admin/custom-providers/qa", { method: "DELETE" });assert.equal(r.status, 200);- assert.equal(resolveModel("qa-chat"), undefined, "model gone after delete");+ assert.equal(resolveModel("custom/qa/qa-chat"), undefined, "model gone after delete");r = await api("/v1/admin/custom-providers/qa", { method: "DELETE" });assert.equal(r.status, 404, "second delete 404s");@@ -228,7 +228,7 @@ test("QA: openai-responses custom provider serves a real turn through the generaassert.equal(validation?.authorization, "Bearer sk-responses-test");assert.equal(((await response.json()) as { status: { protocol: string } }).status.protocol, "openai-responses");- const model = resolveModel("responses-custom-model");+ const model = resolveModel("custom/responses-provider/responses-custom-model");assert.ok(model);assert.equal(model.api, "openai-responses");const reply = await oneShot(@@ -337,7 +337,7 @@ test("QA: anthropic-protocol custom provider serves a real turn (correct wire sh}),});assert.equal(r.status, 200, "anthropic-protocol registration validates against /v1/models with x-api-key");- const model = resolveModel("claude-compat");+ const model = resolveModel("custom/antcompat/claude-compat");assert.ok(model);assert.equal((model as { api?: string }).api, "anthropic-messages");const reply = await oneShot("qa-ant", model as unknown as Model<Api>, { antcompat: "sk-ant-qa" }, "terse", "go");@@ -377,7 +377,7 @@ test("QA: registrations survive a restart (shared durable backing + same secret)assert.equal(await second.resolveKey("survivor"), "sk-live-key", "key decrypts after restart with the same secret");// and the hydration path wires it into the runtime registrysetCustomProviders(enabled);- assert.ok(resolveModel("survivor-model"), "hydrated model resolves");+ assert.ok(resolveModel("custom/survivor/survivor-model"), "hydrated model resolves");setCustomProviders([]);});@@ -404,3 +404,64 @@ test("QA: a corrupt stored key degrades that provider only — admin surface staconst enabled = await reader.enabled();assert.equal(enabled[0]?.id, "corrupted", "spec listing unaffected — only the key is lost");});++test("QA: a custom model named like a built-in registers and sends its own model id upstream", async () => {+ const seen: Array<{ model?: string; auth?: string }> = [];+ const upstream = createServer((req, res) => {+ let body = "";+ req.on("data", (c) => (body += c));+ req.on("end", () => {+ if (!req.url?.endsWith("/chat/completions")) {+ res.writeHead(404);+ return res.end();+ }+ seen.push({ model: (JSON.parse(body) as { model?: string }).model, auth: req.headers.authorization });+ res.writeHead(200, { "content-type": "text/event-stream" });+ const chunk = (delta: object, finish: string | null) =>+ `data: ${JSON.stringify({ id: "cmpl-own", object: "chat.completion.chunk", model: "claude-opus-5", choices: [{ index: 0, delta, finish_reason: finish }], usage: finish ? { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } : undefined })}\n\n`;+ res.write(chunk({ role: "assistant", content: "OWN KEY REPLY" }, null));+ res.write(chunk({}, "stop"));+ res.write("data: [DONE]\n\n");+ res.end();+ });+ });+ await new Promise<void>((r) => upstream.listen(0, "127.0.0.1", r));+ const upstreamUrl = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}/v1`;+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "qa-custom-own-")) }));+ const server = createInsecureTestServer(built.app, {+ config: built.config,+ modelCredentials: built.modelCredentials,+ customProviders: built.customProviders,+ refreshCustomProviders: built.refreshCustomProviders,+ admin: built.admin,+ auditLog: built.auditLog,+ harnessId: "pi",+ });+ server.listen(0);+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;+ try {+ const r = await fetch(`${base}/v1/admin/custom-providers/own`, {+ method: "PUT",+ headers: ADMIN,+ body: JSON.stringify({+ name: "Own endpoint",+ protocol: "openai",+ baseUrl: upstreamUrl,+ apiKey: "sk-own",+ validate: false,+ models: [{ id: "claude-opus-5", name: "Claude Opus Mine" }],+ }),+ });+ assert.equal(r.status, 200);+ const model = resolveModel("custom/own/claude-opus-5");+ assert.equal(model?.provider, "own");+ assert.equal(model?.name, "Claude Opus Mine");+ assert.equal(String(resolveModel("claude-opus-5")?.provider), "anthropic");+ const reply = await oneShot("qa", model as unknown as Model<Api>, { own: "sk-own" }, "you are terse", "hi");+ assert.equal(reply, "OWN KEY REPLY");+ assert.deepEqual(seen, [{ model: "claude-opus-5", auth: "Bearer sk-own" }]);+ } finally {+ server.close();+ upstream.close();+ }+});diff --git a/test/custom-provider-route.test.ts b/test/custom-provider-route.test.tsindex 910bdd9e6..9f0945c46 100644--- a/test/custom-provider-route.test.ts+++ b/test/custom-provider-route.test.ts@@ -72,7 +72,7 @@ test("custom provider lifecycle: register, list, resolve, delete — admin only,assert.equal(JSON.stringify(putBody).includes("sk-acme-secret"), false);// The runtime registry serves the model immediately.- assert.equal(String(resolveModel("acme-large")?.provider), "acme-gateway");+ assert.equal(String(resolveModel("custom/acme-gateway/acme-large")?.provider), "acme-gateway");// List never leaks the key.const list = await fetch(`${srv.base}/v1/admin/custom-providers`, { headers: ADMIN });@@ -91,7 +91,7 @@ test("custom provider lifecycle: register, list, resolve, delete — admin only,headers: ADMIN,});assert.equal(del.status, 200);- assert.equal(resolveModel("acme-large"), undefined);+ assert.equal(resolveModel("custom/acme-gateway/acme-large"), undefined);} finally {await srv.close();}diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.tsindex 5331b5173..e26637e64 100644--- a/test/custom-providers.test.ts+++ b/test/custom-providers.test.ts@@ -26,7 +26,7 @@ const GATEWAY = {test("a registered custom model resolves with the provider's protocol and base URL", () => {setCustomProviders([GATEWAY]);- const model = resolveCustomModel("acme-large");+ const model = resolveCustomModel("custom/acme-gateway/acme-large");assert.ok(model);assert.equal(model.provider, "acme-gateway");assert.equal(model.api, "openai-completions");@@ -45,7 +45,7 @@ test("anthropic-protocol providers produce anthropic-messages models with defaulmodels: [{ id: "eu-claude" }],},]);- const model = resolveCustomModel("eu-claude");+ const model = resolveCustomModel("custom/eu-anthropic/eu-claude");assert.ok(model);assert.equal(model.api, "anthropic-messages");assert.equal(model.contextWindow, 128_000);@@ -62,36 +62,41 @@ test("openai-responses survives both custom model mappings", () => {models: [{ id: "responses-model" }],},]);- assert.equal(resolveCustomModel("responses-model")?.api, "openai-responses");+ assert.equal(resolveCustomModel("custom/responses-gateway/responses-model")?.api, "openai-responses");const generated = customModelsJson() as { providers: Record<string, { api: string }> };assert.equal(generated.providers["responses-gateway"]?.api, "openai-responses");});-test("resolveModel falls back to custom models; built-ins shadow custom ids", () => {+test("custom models resolve in their own namespace, so they never shadow a built-in of the same name", () => {setCustomProviders([{ ...GATEWAY, models: [{ id: "acme-large" }, { id: "claude-opus-5", name: "impostor" }] }]);- assert.equal(resolveModel("acme-large")?.provider, "acme-gateway");- // The built-in claude-opus-5 must win over a custom model claiming its id.+ assert.equal(resolveModel("custom/acme-gateway/acme-large")?.provider, "acme-gateway");+ assert.equal(resolveModel("custom/acme-gateway/claude-opus-5")?.name, "impostor");assert.equal(String(resolveModel("claude-opus-5")?.provider), "anthropic");});test("custom models are gated to pi and mock harnesses", () => {setCustomProviders([GATEWAY]);- assert.equal(modelSupportedByHarness("acme-large", "pi"), true);- assert.equal(modelSupportedByHarness("acme-large", "mock"), true);- assert.equal(modelSupportedByHarness("acme-large", "claude"), false);+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "pi"), true);+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "mock"), true);+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "claude"), false);});test("a registered custom model is serviceable regardless of built-in key availability", () => {setCustomProviders([GATEWAY]);- assert.equal(modelServiceable("acme-large", { anthropic: false, openai: false, openrouter: false }), true);+ assert.equal(+ modelServiceable("custom/acme-gateway/acme-large", { anthropic: false, openai: false, openrouter: false }),+ true,+ );});test("catalog lists custom models; clearing the registry removes them", () => {setCustomProviders([GATEWAY]);- assert.deepEqual(customModelCatalog(), [{ id: "acme-large", name: "Acme Large", provider: "acme-gateway" }]);+ assert.deepEqual(customModelCatalog(), [+ { id: "custom/acme-gateway/acme-large", name: "Acme Large", provider: "acme-gateway" },+ ]);setCustomProviders([]);- assert.equal(isCustomModelId("acme-large"), false);- assert.equal(resolveModel("acme-large"), undefined);+ assert.equal(isCustomModelId("custom/acme-gateway/acme-large"), false);+ assert.equal(resolveModel("custom/acme-gateway/acme-large"), undefined);});test("spec validation rejects reserved ids, bad slugs, bad URLs, and empty model lists", () => {@@ -150,11 +155,11 @@ test("registered models surface in the catalog and vanish on unregister", () =>},]);const catalog = builtInModelCatalog();- const entry = catalog.find((m) => m.id === "deepseek-chat");+ const entry = catalog.find((m) => m.id === "custom/deepseek/deepseek-chat");assert.ok(entry, "custom model appears in the catalog");assert.equal(entry!.provider, "deepseek");setCustomProviders([]);- assert.ok(!builtInModelCatalog().some((m) => m.id === "deepseek-chat"));+ assert.ok(!builtInModelCatalog().some((m) => m.id === "custom/deepseek/deepseek-chat"));});test("catalog cache invalidates immediately when the custom registry changes", async () => {@@ -162,7 +167,7 @@ test("catalog cache invalidates immediately when the custom registry changes", aconst fetcher: typeof fetch = async () => new Response(JSON.stringify({ data: [] }), { status: 200 });setCustomProviders([]);const before = await selectableModelCatalog(fetcher);- assert.ok(!before.some((m) => m.id === "fresh-model"));+ assert.ok(!before.some((m) => m.id === "custom/freshco/fresh-model"));setCustomProviders([{id: "freshco",@@ -175,14 +180,14 @@ test("catalog cache invalidates immediately when the custom registry changes", atry {const after = await selectableModelCatalog(fetcher);assert.ok(- after.some((m) => m.id === "fresh-model"),+ after.some((m) => m.id === "custom/freshco/fresh-model"),"new registration visible without waiting out the TTL",);} finally {setCustomProviders([]);}const cleared = await selectableModelCatalog(fetcher);- assert.ok(!cleared.some((m) => m.id === "fresh-model"), "removal visible immediately too");+ assert.ok(!cleared.some((m) => m.id === "custom/freshco/fresh-model"), "removal visible immediately too");});for (const field of ["contextWindow", "maxTokens"] as const) {@@ -212,7 +217,7 @@ for (const protocol of ["openai", "openai-responses", "anthropic"] as const) {await store.upsert(spec, "local", "[email protected]");const reloaded = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });setCustomProviders(await reloaded.enabled());- const model = resolveCustomModel("acme-large")!;+ const model = resolveCustomModel("custom/acme-gateway/acme-large")!;assert.equal(model.reasoning, reasoning ?? false);const json = customModelsJson() as {providers: Record<string, { models: Array<{ reasoning: boolean; cost: typeof model.cost }> }>;diff --git a/test/gateway-catalog.test.ts b/test/gateway-catalog.test.tsindex c50bf96af..d90f701df 100644--- a/test/gateway-catalog.test.ts+++ b/test/gateway-catalog.test.ts@@ -17,7 +17,7 @@ import {safeModelMetadata,} from "../src/model/pi-models.ts";import { builtInModelCatalog } from "../src/model/model-catalog.ts";-import { validateCustomProviderSpec } from "../src/model/custom-providers.ts";+import { setCustomProviders, validateCustomProviderSpec } from "../src/model/custom-providers.ts";import { loadConfig } from "../src/config.ts";const config = {@@ -209,7 +209,7 @@ test("malformed and oversized catalogs cannot grant access", async () => {}});-test("gateway namespace cannot be registered by custom providers even before discovery", () => {+test("custom provider models live in their own namespace, never the gateway one", () => {assert.doesNotThrow(() =>validateCustomProviderSpec({id: "gateway",@@ -227,7 +227,11 @@ test("gateway namespace cannot be registered by custom providers even before disbaseUrl: config.url,models: [{ id: "gateway/future" }],};- assert.throws(() => validateCustomProviderSpec(spec), /already registered/);+ assert.doesNotThrow(() => validateCustomProviderSpec(spec));+ setCustomProviders([spec]);+ assert.equal(resolveModel("custom/custom/gateway/future")?.provider, "custom");+ assert.equal(resolveModel("gateway/future"), undefined);+ setCustomProviders([]);assert.throws(() => validateCustomProviderSpec({ ...spec, id: "qm:gateway", models: [{ id: "other" }] }),/must match/,diff --git a/test/model-overlay-postgres.test.ts b/test/model-overlay-postgres.test.tsindex 660f9ef63..9fad08235 100644--- a/test/model-overlay-postgres.test.ts+++ b/test/model-overlay-postgres.test.ts@@ -113,10 +113,12 @@ test(]);assert.deepEqual(raced.map((response) => response.status),- raced[0]!.status === 200 ? [200, 400] : [409, 200],+ [200, 200],);await read(first.base);- await read(second.base);+ const both = (await read(second.base)).modelCatalog;+ assert.ok(both["overlay-race-model"]);+ assert.equal(both["custom/race-gateway/overlay-race-model"]?.name, "overlay-race-model");const cold = await start(isolatedDatabaseUrl, upstream.url);processes.push(cold.child);const coldConfig = await read(cold.base);diff --git a/test/model-overlay.test.ts b/test/model-overlay.test.tsindex 3d68365af..5eda11175 100644--- a/test/model-overlay.test.ts+++ b/test/model-overlay.test.ts@@ -55,7 +55,7 @@ import {fastModeModelIds,defaultWebuiModelIds,} from "../src/model/pi-models.ts";-import { setCustomProviders, validateCustomProviderSpec } from "../src/model/custom-providers.ts";+import { setCustomProviders } from "../src/model/custom-providers.ts";import { selectableModelCatalog } from "../src/model/model-catalog.ts";import { resolveIndividualAuthRouting } from "../src/core/individual-auth-routing.ts";import { validateWebTurnModelOptions } from "../src/core/turn-options.ts";@@ -177,17 +177,7 @@ test("template capabilities are inherited and overlay API models cannot silently);});-test("collisions are rejected in both registration orders and OpenRouter failure rebuilds local choices", async () => {- const custom = {- id: "gateway",- name: "Gateway",- protocol: "openai" as const,- baseUrl: "https://example.invalid/v1",- models: [{ id: MODEL_ID }],- };- setCustomProviders([custom]);- assert.throws(() => validateModelOverlay(spec), /already registered/);- setCustomProviders([]);+test("OpenRouter failure rebuilds local choices", async () => {let fail = false;const fetcher: typeof fetch = async () => {if (fail) throw new Error("offline");@@ -195,12 +185,10 @@ test("collisions are rejected in both registration orders and OpenRouter failure};await selectableModelCatalog(fetcher);setModelOverlays([validateModelOverlay(spec)]);- assert.throws(() => validateCustomProviderSpec(custom), /already registered/);fail = true;assert.ok((await selectableModelCatalog(fetcher)).some((m) => m.id === MODEL_ID));setModelOverlays([], [MODEL_ID]);assert.ok(!(await selectableModelCatalog(fetcher)).some((m) => m.id === MODEL_ID));- assert.throws(() => validateCustomProviderSpec(custom), /already registered/);});test("live admin lifecycle is authorized, audited, immediately selectable and removed on delete", async () => {