Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Let custom providers serve models QM already ships, under their own name (#2427)

Joshua France9eda1297e2b9
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 79396c662..6b5257267 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -75,7 +75,7 @@ import {
codexProviderModelId,
modelHasPrices,
} from "../model/pi-models.ts";
-import { customModelsJson, customProvidersVersion } from "../model/custom-providers.ts";
+import { customModelsJson, customProvidersVersion, resolveCustomModel } from "../model/custom-providers.ts";
import {
GatewayModelUnavailableError,
modelGatewayRequest,
@@ -1204,7 +1204,7 @@ export async function buildModelRuntime(
const providerModelId =
model.provider === CODEX_SUBSCRIPTION_PROVIDER
? codexProviderModelId(model.id)
- : (modelRequestOverrides(model.id)?.model ?? model.id);
+ : (modelRequestOverrides(model.id)?.model ?? resolveCustomModel(model.id)?.upstreamId ?? model.id);
const candidate = withRequestHeaders(model, true, false);
const passthrough = {
...retained(options),
diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.ts
index 442cdb84e..858d7ed2b 100644
--- a/src/model/custom-providers.ts
+++ b/src/model/custom-providers.ts
@@ -13,7 +13,6 @@
* resolved per-call by wiring alongside the built-in provider keys.
*/
-import { modelIdReserved } from "./pi-models.ts";
import { parseProviderBaseUrl, PROVIDER_IDS } from "./provider-endpoints.ts";
export const CUSTOM_PROVIDER_PROTOCOLS = ["openai", "openai-responses", "anthropic"] as const;
@@ -65,8 +64,6 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {
throw new Error(`model "${m.id}": name must be a string of 200 chars or fewer`);
if (m.reasoning !== undefined && typeof m.reasoning !== "boolean")
throw new Error(`model "${m.id}": reasoning must be a boolean`);
- if (modelIdReserved(m.id) || (registry.has(m.id) && registry.get(m.id)?.provider !== spec.id))
- throw new Error(`model id "${m.id}" is already registered`);
if (seen.has(m.id)) throw new Error(`duplicate model id "${m.id}"`);
seen.add(m.id);
for (const [field, v] of [
@@ -95,6 +92,7 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {
*/
export interface CustomRuntimeModel {
id: string;
+ upstreamId: string;
name: string;
provider: string;
api: "openai-completions" | "openai-responses" | "anthropic-messages";
@@ -117,7 +115,8 @@ function customProviderApi(protocol: CustomProviderProtocol): CustomRuntimeModel
function toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): CustomRuntimeModel {
return {
- id: m.id,
+ id: `custom/${provider.id}/${m.id}`,
+ upstreamId: m.id,
name: m.name?.trim() || m.id,
provider: provider.id,
api: customProviderApi(provider.protocol),
@@ -146,7 +145,8 @@ export function setCustomProviders(specs: CustomProviderSpec[]): void {
const next = new Map<string, CustomRuntimeModel>();
for (const spec of specs) {
for (const m of spec.models) {
- next.set(m.id, toRuntimeModel(spec, m));
+ const model = toRuntimeModel(spec, m);
+ next.set(model.id, model);
}
}
registry = next;
@@ -188,14 +188,16 @@ export function customModelsJson(): { providers: Record<string, unknown> } | und
name: spec.name,
baseUrl: spec.baseUrl,
api: customProviderApi(spec.protocol),
- models: spec.models.map((m) => ({
- id: m.id,
- name: m.name ?? m.id,
- reasoning: m.reasoning ?? false,
- contextWindow: m.contextWindow ?? 128_000,
- maxTokens: m.maxTokens ?? 8_192,
- cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },
- })),
+ models: [...registry.values()]
+ .filter((m) => m.provider === spec.id)
+ .map(({ id, name, reasoning, contextWindow, maxTokens, cost }) => ({
+ id,
+ name,
+ reasoning,
+ contextWindow,
+ maxTokens,
+ cost,
+ })),
},
]),
),
diff --git a/src/model/pi-models.ts b/src/model/pi-models.ts
index 7ea212cf3..8100003a4 100644
--- a/src/model/pi-models.ts
+++ b/src/model/pi-models.ts
@@ -415,12 +415,7 @@ export function modelIdReserved(id: string): boolean {
export function validateModelOverlay(value: unknown): RegisteredModel {
const spec = parseRegisteredModel(value);
- if (
- REGISTRY_BY_ID.has(spec.id) ||
- builtinModel(spec.id) ||
- isCustomModelId(spec.id) ||
- OPENROUTER_CATALOG_MODELS.has(spec.id)
- )
+ if (REGISTRY_BY_ID.has(spec.id) || builtinModel(spec.id) || OPENROUTER_CATALOG_MODELS.has(spec.id))
throw new Error("model id is already registered");
if (isEmbeddingModel(spec)) return spec;
const template = resolveBuiltinModel(spec.template);
@@ -450,7 +445,7 @@ export function setModelOverlays(
}
if (isEmbeddingModel(spec)) {
if (verificationFailures.has(id)) unavailable.set(id, verificationFailures.get(id)!);
- else if (!isCustomModelId(id)) next.set(id, spec);
+ else next.set(id, spec);
continue;
}
const template = resolveBuiltinModel(spec.template);
@@ -459,7 +454,7 @@ export function setModelOverlays(
id,
"Builtin template is unavailable or has a different provider; ask an administrator to repair this model",
);
- } else if (isCustomModelId(id) || OPENROUTER_CATALOG_MODELS.has(id)) {
+ } else if (OPENROUTER_CATALOG_MODELS.has(id)) {
unavailable.set(id, "Model id conflicts with another provider; ask an administrator to repair this model");
} else if (verificationFailures.has(id)) unavailable.set(id, verificationFailures.get(id)!);
else next.set(id, spec);
diff --git a/test/custom-provider-boot-wiring.test.ts b/test/custom-provider-boot-wiring.test.ts
index 3c7730c40..1bf21e821 100644
--- a/test/custom-provider-boot-wiring.test.ts
+++ b/test/custom-provider-boot-wiring.test.ts
@@ -20,19 +20,19 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defa
const config = testConfig({
dataDir: mkdtempSync(join(tmpdir(), "custom-provider-boot-")),
harness: "pi",
- modelId: "acme-large",
+ modelId: "custom/acme-gateway/acme-large",
});
const built = buildApp(config, { modelCredentialFetch: async () => new Response(null, { status: 200 }) });
const deps = serverDeps(config, built);
assert.equal(deps.customProviders, built.customProviders);
assert.equal(deps.refreshCustomProviders, built.refreshCustomProviders);
- assert.equal(deps.baseModelDefault, "acme-large");
+ assert.equal(deps.baseModelDefault, "custom/acme-gateway/acme-large");
const server = createInsecureTestServer(built.app, deps);
server.listen(0);
const base = `http://localhost:${(server.address() as AddressInfo).port}`;
try {
- assert.notEqual(defaultModelForHarness("pi", deps.baseModelDefault), "acme-large");
+ assert.notEqual(defaultModelForHarness("pi", deps.baseModelDefault), "custom/acme-gateway/acme-large");
const list = await fetch(`${base}/v1/admin/custom-providers`, { headers: ADMIN });
assert.equal(list.status, 200);
@@ -51,7 +51,7 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defa
});
assert.equal(put.status, 200);
- assert.equal(defaultModelForHarness("pi", deps.baseModelDefault), "acme-large");
+ assert.equal(defaultModelForHarness("pi", deps.baseModelDefault), "custom/acme-gateway/acme-large");
const runtime = await fetch(
`${base}/v1/runtime-config?principalId=admin-alice@default-org&scopeId=personal:admin-alice@default-org`,
@@ -63,9 +63,9 @@ test("serverDeps wires the custom-provider store and resolves a custom boot defa
modelsByHarness: Record<string, string[]>;
modelCatalog: Record<string, { name: string; provider: string }>;
};
- assert.equal(body.effective.modelId, "acme-large");
- assert.ok(body.modelsByHarness.pi?.includes("acme-large"));
- assert.equal(body.modelCatalog["acme-large"]?.provider, "acme-gateway");
+ assert.equal(body.effective.modelId, "custom/acme-gateway/acme-large");
+ assert.ok(body.modelsByHarness.pi?.includes("custom/acme-gateway/acme-large"));
+ assert.equal(body.modelCatalog["custom/acme-gateway/acme-large"]?.provider, "acme-gateway");
} finally {
await new Promise<void>((resolve) => server.close(() => resolve()));
}
diff --git a/test/custom-provider-e2e.test.ts b/test/custom-provider-e2e.test.ts
index bb3c8b757..bf54ad4cd 100644
--- a/test/custom-provider-e2e.test.ts
+++ b/test/custom-provider-e2e.test.ts
@@ -144,13 +144,13 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()
assert.ok(!listing.includes("sk-qa-good"), "key never readable");
// 5. model resolves like a built-in and is catalog-visible
- const model = resolveModel("qa-chat");
+ const model = resolveModel("custom/qa/qa-chat");
assert.ok(model, "custom model resolves");
assert.equal(model!.provider, "qa");
assert.equal(model!.reasoning, true);
assert.equal((model as { baseUrl?: string }).baseUrl, upstreamUrl);
- assert.equal(modelSupportedByHarness("qa-chat", "pi"), true);
- assert.equal(modelServiceable("qa-chat", { anthropic: false, openai: false, openrouter: false }), true);
+ assert.equal(modelSupportedByHarness("custom/qa/qa-chat", "pi"), true);
+ assert.equal(modelServiceable("custom/qa/qa-chat", { anthropic: false, openai: false, openrouter: false }), true);
// 6. REAL model call through QM's pi path → fake upstream answers
const reply = await oneShot(
@@ -183,7 +183,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()
// 8. delete: models leave the registry
r = await api("/v1/admin/custom-providers/qa", { method: "DELETE" });
assert.equal(r.status, 200);
- assert.equal(resolveModel("qa-chat"), undefined, "model gone after delete");
+ assert.equal(resolveModel("custom/qa/qa-chat"), undefined, "model gone after delete");
r = await api("/v1/admin/custom-providers/qa", { method: "DELETE" });
assert.equal(r.status, 404, "second delete 404s");
@@ -228,7 +228,7 @@ test("QA: openai-responses custom provider serves a real turn through the genera
assert.equal(validation?.authorization, "Bearer sk-responses-test");
assert.equal(((await response.json()) as { status: { protocol: string } }).status.protocol, "openai-responses");
- const model = resolveModel("responses-custom-model");
+ const model = resolveModel("custom/responses-provider/responses-custom-model");
assert.ok(model);
assert.equal(model.api, "openai-responses");
const reply = await oneShot(
@@ -337,7 +337,7 @@ test("QA: anthropic-protocol custom provider serves a real turn (correct wire sh
}),
});
assert.equal(r.status, 200, "anthropic-protocol registration validates against /v1/models with x-api-key");
- const model = resolveModel("claude-compat");
+ const model = resolveModel("custom/antcompat/claude-compat");
assert.ok(model);
assert.equal((model as { api?: string }).api, "anthropic-messages");
const reply = await oneShot("qa-ant", model as unknown as Model<Api>, { antcompat: "sk-ant-qa" }, "terse", "go");
@@ -377,7 +377,7 @@ test("QA: registrations survive a restart (shared durable backing + same secret)
assert.equal(await second.resolveKey("survivor"), "sk-live-key", "key decrypts after restart with the same secret");
// and the hydration path wires it into the runtime registry
setCustomProviders(enabled);
- assert.ok(resolveModel("survivor-model"), "hydrated model resolves");
+ assert.ok(resolveModel("custom/survivor/survivor-model"), "hydrated model resolves");
setCustomProviders([]);
});
@@ -404,3 +404,64 @@ test("QA: a corrupt stored key degrades that provider only — admin surface sta
const enabled = await reader.enabled();
assert.equal(enabled[0]?.id, "corrupted", "spec listing unaffected — only the key is lost");
});
+
+test("QA: a custom model named like a built-in registers and sends its own model id upstream", async () => {
+ const seen: Array<{ model?: string; auth?: string }> = [];
+ const upstream = createServer((req, res) => {
+ let body = "";
+ req.on("data", (c) => (body += c));
+ req.on("end", () => {
+ if (!req.url?.endsWith("/chat/completions")) {
+ res.writeHead(404);
+ return res.end();
+ }
+ seen.push({ model: (JSON.parse(body) as { model?: string }).model, auth: req.headers.authorization });
+ res.writeHead(200, { "content-type": "text/event-stream" });
+ const chunk = (delta: object, finish: string | null) =>
+ `data: ${JSON.stringify({ id: "cmpl-own", object: "chat.completion.chunk", model: "claude-opus-5", choices: [{ index: 0, delta, finish_reason: finish }], usage: finish ? { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } : undefined })}\n\n`;
+ res.write(chunk({ role: "assistant", content: "OWN KEY REPLY" }, null));
+ res.write(chunk({}, "stop"));
+ res.write("data: [DONE]\n\n");
+ res.end();
+ });
+ });
+ await new Promise<void>((r) => upstream.listen(0, "127.0.0.1", r));
+ const upstreamUrl = `http://127.0.0.1:${(upstream.address() as AddressInfo).port}/v1`;
+ const built = buildApp(testConfig({ dataDir: mkdtempSync(join(tmpdir(), "qa-custom-own-")) }));
+ const server = createInsecureTestServer(built.app, {
+ config: built.config,
+ modelCredentials: built.modelCredentials,
+ customProviders: built.customProviders,
+ refreshCustomProviders: built.refreshCustomProviders,
+ admin: built.admin,
+ auditLog: built.auditLog,
+ harnessId: "pi",
+ });
+ server.listen(0);
+ const base = `http://localhost:${(server.address() as AddressInfo).port}`;
+ try {
+ const r = await fetch(`${base}/v1/admin/custom-providers/own`, {
+ method: "PUT",
+ headers: ADMIN,
+ body: JSON.stringify({
+ name: "Own endpoint",
+ protocol: "openai",
+ baseUrl: upstreamUrl,
+ apiKey: "sk-own",
+ validate: false,
+ models: [{ id: "claude-opus-5", name: "Claude Opus Mine" }],
+ }),
+ });
+ assert.equal(r.status, 200);
+ const model = resolveModel("custom/own/claude-opus-5");
+ assert.equal(model?.provider, "own");
+ assert.equal(model?.name, "Claude Opus Mine");
+ assert.equal(String(resolveModel("claude-opus-5")?.provider), "anthropic");
+ const reply = await oneShot("qa", model as unknown as Model<Api>, { own: "sk-own" }, "you are terse", "hi");
+ assert.equal(reply, "OWN KEY REPLY");
+ assert.deepEqual(seen, [{ model: "claude-opus-5", auth: "Bearer sk-own" }]);
+ } finally {
+ server.close();
+ upstream.close();
+ }
+});
diff --git a/test/custom-provider-route.test.ts b/test/custom-provider-route.test.ts
index 910bdd9e6..9f0945c46 100644
--- a/test/custom-provider-route.test.ts
+++ b/test/custom-provider-route.test.ts
@@ -72,7 +72,7 @@ test("custom provider lifecycle: register, list, resolve, delete — admin only,
assert.equal(JSON.stringify(putBody).includes("sk-acme-secret"), false);
// The runtime registry serves the model immediately.
- assert.equal(String(resolveModel("acme-large")?.provider), "acme-gateway");
+ assert.equal(String(resolveModel("custom/acme-gateway/acme-large")?.provider), "acme-gateway");
// List never leaks the key.
const list = await fetch(`${srv.base}/v1/admin/custom-providers`, { headers: ADMIN });
@@ -91,7 +91,7 @@ test("custom provider lifecycle: register, list, resolve, delete — admin only,
headers: ADMIN,
});
assert.equal(del.status, 200);
- assert.equal(resolveModel("acme-large"), undefined);
+ assert.equal(resolveModel("custom/acme-gateway/acme-large"), undefined);
} finally {
await srv.close();
}
diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.ts
index 5331b5173..e26637e64 100644
--- a/test/custom-providers.test.ts
+++ b/test/custom-providers.test.ts
@@ -26,7 +26,7 @@ const GATEWAY = {
test("a registered custom model resolves with the provider's protocol and base URL", () => {
setCustomProviders([GATEWAY]);
- const model = resolveCustomModel("acme-large");
+ const model = resolveCustomModel("custom/acme-gateway/acme-large");
assert.ok(model);
assert.equal(model.provider, "acme-gateway");
assert.equal(model.api, "openai-completions");
@@ -45,7 +45,7 @@ test("anthropic-protocol providers produce anthropic-messages models with defaul
models: [{ id: "eu-claude" }],
},
]);
- const model = resolveCustomModel("eu-claude");
+ const model = resolveCustomModel("custom/eu-anthropic/eu-claude");
assert.ok(model);
assert.equal(model.api, "anthropic-messages");
assert.equal(model.contextWindow, 128_000);
@@ -62,36 +62,41 @@ test("openai-responses survives both custom model mappings", () => {
models: [{ id: "responses-model" }],
},
]);
- assert.equal(resolveCustomModel("responses-model")?.api, "openai-responses");
+ assert.equal(resolveCustomModel("custom/responses-gateway/responses-model")?.api, "openai-responses");
const generated = customModelsJson() as { providers: Record<string, { api: string }> };
assert.equal(generated.providers["responses-gateway"]?.api, "openai-responses");
});
-test("resolveModel falls back to custom models; built-ins shadow custom ids", () => {
+test("custom models resolve in their own namespace, so they never shadow a built-in of the same name", () => {
setCustomProviders([{ ...GATEWAY, models: [{ id: "acme-large" }, { id: "claude-opus-5", name: "impostor" }] }]);
- assert.equal(resolveModel("acme-large")?.provider, "acme-gateway");
- // The built-in claude-opus-5 must win over a custom model claiming its id.
+ assert.equal(resolveModel("custom/acme-gateway/acme-large")?.provider, "acme-gateway");
+ assert.equal(resolveModel("custom/acme-gateway/claude-opus-5")?.name, "impostor");
assert.equal(String(resolveModel("claude-opus-5")?.provider), "anthropic");
});
test("custom models are gated to pi and mock harnesses", () => {
setCustomProviders([GATEWAY]);
- assert.equal(modelSupportedByHarness("acme-large", "pi"), true);
- assert.equal(modelSupportedByHarness("acme-large", "mock"), true);
- assert.equal(modelSupportedByHarness("acme-large", "claude"), false);
+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "pi"), true);
+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "mock"), true);
+ assert.equal(modelSupportedByHarness("custom/acme-gateway/acme-large", "claude"), false);
});
test("a registered custom model is serviceable regardless of built-in key availability", () => {
setCustomProviders([GATEWAY]);
- assert.equal(modelServiceable("acme-large", { anthropic: false, openai: false, openrouter: false }), true);
+ assert.equal(
+ modelServiceable("custom/acme-gateway/acme-large", { anthropic: false, openai: false, openrouter: false }),
+ true,
+ );
});
test("catalog lists custom models; clearing the registry removes them", () => {
setCustomProviders([GATEWAY]);
- assert.deepEqual(customModelCatalog(), [{ id: "acme-large", name: "Acme Large", provider: "acme-gateway" }]);
+ assert.deepEqual(customModelCatalog(), [
+ { id: "custom/acme-gateway/acme-large", name: "Acme Large", provider: "acme-gateway" },
+ ]);
setCustomProviders([]);
- assert.equal(isCustomModelId("acme-large"), false);
- assert.equal(resolveModel("acme-large"), undefined);
+ assert.equal(isCustomModelId("custom/acme-gateway/acme-large"), false);
+ assert.equal(resolveModel("custom/acme-gateway/acme-large"), undefined);
});
test("spec validation rejects reserved ids, bad slugs, bad URLs, and empty model lists", () => {
@@ -150,11 +155,11 @@ test("registered models surface in the catalog and vanish on unregister", () =>
},
]);
const catalog = builtInModelCatalog();
- const entry = catalog.find((m) => m.id === "deepseek-chat");
+ const entry = catalog.find((m) => m.id === "custom/deepseek/deepseek-chat");
assert.ok(entry, "custom model appears in the catalog");
assert.equal(entry!.provider, "deepseek");
setCustomProviders([]);
- assert.ok(!builtInModelCatalog().some((m) => m.id === "deepseek-chat"));
+ assert.ok(!builtInModelCatalog().some((m) => m.id === "custom/deepseek/deepseek-chat"));
});
test("catalog cache invalidates immediately when the custom registry changes", async () => {
@@ -162,7 +167,7 @@ test("catalog cache invalidates immediately when the custom registry changes", a
const fetcher: typeof fetch = async () => new Response(JSON.stringify({ data: [] }), { status: 200 });
setCustomProviders([]);
const before = await selectableModelCatalog(fetcher);
- assert.ok(!before.some((m) => m.id === "fresh-model"));
+ assert.ok(!before.some((m) => m.id === "custom/freshco/fresh-model"));
setCustomProviders([
{
id: "freshco",
@@ -175,14 +180,14 @@ test("catalog cache invalidates immediately when the custom registry changes", a
try {
const after = await selectableModelCatalog(fetcher);
assert.ok(
- after.some((m) => m.id === "fresh-model"),
+ after.some((m) => m.id === "custom/freshco/fresh-model"),
"new registration visible without waiting out the TTL",
);
} finally {
setCustomProviders([]);
}
const cleared = await selectableModelCatalog(fetcher);
- assert.ok(!cleared.some((m) => m.id === "fresh-model"), "removal visible immediately too");
+ assert.ok(!cleared.some((m) => m.id === "custom/freshco/fresh-model"), "removal visible immediately too");
});
for (const field of ["contextWindow", "maxTokens"] as const) {
@@ -212,7 +217,7 @@ for (const protocol of ["openai", "openai-responses", "anthropic"] as const) {
await store.upsert(spec, "local", "[email protected]");
const reloaded = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });
setCustomProviders(await reloaded.enabled());
- const model = resolveCustomModel("acme-large")!;
+ const model = resolveCustomModel("custom/acme-gateway/acme-large")!;
assert.equal(model.reasoning, reasoning ?? false);
const json = customModelsJson() as {
providers: Record<string, { models: Array<{ reasoning: boolean; cost: typeof model.cost }> }>;
diff --git a/test/gateway-catalog.test.ts b/test/gateway-catalog.test.ts
index c50bf96af..d90f701df 100644
--- a/test/gateway-catalog.test.ts
+++ b/test/gateway-catalog.test.ts
@@ -17,7 +17,7 @@ import {
safeModelMetadata,
} from "../src/model/pi-models.ts";
import { builtInModelCatalog } from "../src/model/model-catalog.ts";
-import { validateCustomProviderSpec } from "../src/model/custom-providers.ts";
+import { setCustomProviders, validateCustomProviderSpec } from "../src/model/custom-providers.ts";
import { loadConfig } from "../src/config.ts";
const config = {
@@ -209,7 +209,7 @@ test("malformed and oversized catalogs cannot grant access", async () => {
}
});
-test("gateway namespace cannot be registered by custom providers even before discovery", () => {
+test("custom provider models live in their own namespace, never the gateway one", () => {
assert.doesNotThrow(() =>
validateCustomProviderSpec({
id: "gateway",
@@ -227,7 +227,11 @@ test("gateway namespace cannot be registered by custom providers even before dis
baseUrl: config.url,
models: [{ id: "gateway/future" }],
};
- assert.throws(() => validateCustomProviderSpec(spec), /already registered/);
+ assert.doesNotThrow(() => validateCustomProviderSpec(spec));
+ setCustomProviders([spec]);
+ assert.equal(resolveModel("custom/custom/gateway/future")?.provider, "custom");
+ assert.equal(resolveModel("gateway/future"), undefined);
+ setCustomProviders([]);
assert.throws(
() => validateCustomProviderSpec({ ...spec, id: "qm:gateway", models: [{ id: "other" }] }),
/must match/,
diff --git a/test/model-overlay-postgres.test.ts b/test/model-overlay-postgres.test.ts
index 660f9ef63..9fad08235 100644
--- a/test/model-overlay-postgres.test.ts
+++ b/test/model-overlay-postgres.test.ts
@@ -113,10 +113,12 @@ test(
]);
assert.deepEqual(
raced.map((response) => response.status),
- raced[0]!.status === 200 ? [200, 400] : [409, 200],
+ [200, 200],
);
await read(first.base);
- await read(second.base);
+ const both = (await read(second.base)).modelCatalog;
+ assert.ok(both["overlay-race-model"]);
+ assert.equal(both["custom/race-gateway/overlay-race-model"]?.name, "overlay-race-model");
const cold = await start(isolatedDatabaseUrl, upstream.url);
processes.push(cold.child);
const coldConfig = await read(cold.base);
diff --git a/test/model-overlay.test.ts b/test/model-overlay.test.ts
index 3d68365af..5eda11175 100644
--- a/test/model-overlay.test.ts
+++ b/test/model-overlay.test.ts
@@ -55,7 +55,7 @@ import {
fastModeModelIds,
defaultWebuiModelIds,
} from "../src/model/pi-models.ts";
-import { setCustomProviders, validateCustomProviderSpec } from "../src/model/custom-providers.ts";
+import { setCustomProviders } from "../src/model/custom-providers.ts";
import { selectableModelCatalog } from "../src/model/model-catalog.ts";
import { resolveIndividualAuthRouting } from "../src/core/individual-auth-routing.ts";
import { validateWebTurnModelOptions } from "../src/core/turn-options.ts";
@@ -177,17 +177,7 @@ test("template capabilities are inherited and overlay API models cannot silently
);
});
-test("collisions are rejected in both registration orders and OpenRouter failure rebuilds local choices", async () => {
- const custom = {
- id: "gateway",
- name: "Gateway",
- protocol: "openai" as const,
- baseUrl: "https://example.invalid/v1",
- models: [{ id: MODEL_ID }],
- };
- setCustomProviders([custom]);
- assert.throws(() => validateModelOverlay(spec), /already registered/);
- setCustomProviders([]);
+test("OpenRouter failure rebuilds local choices", async () => {
let fail = false;
const fetcher: typeof fetch = async () => {
if (fail) throw new Error("offline");
@@ -195,12 +185,10 @@ test("collisions are rejected in both registration orders and OpenRouter failure
};
await selectableModelCatalog(fetcher);
setModelOverlays([validateModelOverlay(spec)]);
- assert.throws(() => validateCustomProviderSpec(custom), /already registered/);
fail = true;
assert.ok((await selectableModelCatalog(fetcher)).some((m) => m.id === MODEL_ID));
setModelOverlays([], [MODEL_ID]);
assert.ok(!(await selectableModelCatalog(fetcher)).some((m) => m.id === MODEL_ID));
- assert.throws(() => validateCustomProviderSpec(custom), /already registered/);
});
test("live admin lifecycle is authorized, audited, immediately selectable and removed on delete", async () => {