Multiplayer agent harness for work.
Allow reasoning capability on custom models (#1810)
Sina Matian9627efc71109
diff --git a/plugins/admin/test/settings-provider-races.test.ts b/plugins/admin/test/settings-provider-races.test.tsindex cc41d4941..c3d982ea0 100644--- a/plugins/admin/test/settings-provider-races.test.ts+++ b/plugins/admin/test/settings-provider-races.test.ts@@ -78,3 +78,28 @@ test("old provider save cannot clear pending state in a reopened dialog", asyncassert.equal((dom.window.document.getElementById("custom-provider-dialog") as HTMLDialogElement).open, false);dom.window.close();});++test("reasoning edits preserve pricing and omitted positional fields", () => {+ const { dom, ui } = fixture();+ ui.openProvider({+ id: "local",+ name: "Local",+ protocol: "openai",+ baseUrl: "http://localhost:8000/v1",+ models: [+ { id: "thinking", contextWindow: 32768, maxTokens: 4096, reasoning: true, input: 0, output: 2 },+ { id: "plain", name: "Plain", reasoning: false },+ ],+ });+ assert.equal(ui.provider.draft.models, "thinking | | 32768 | 4096 | true\nplain | Plain | | | false");+ assert.deepEqual(JSON.parse(JSON.stringify(ui.providerBody().models)), [+ { id: "thinking", contextWindow: 32768, maxTokens: 4096, reasoning: true, input: 0, output: 2 },+ { id: "plain", name: "Plain", reasoning: false },+ ]);+ ui.provider.draft.models = "thinking | | 32768 | 4096 | false\nnew-model | New | 16000 | 2000 | true";+ const models = ui.providerBody().models;+ assert.equal(models[0].reasoning, false);+ assert.equal(models[0].output, 2);+ assert.equal(models[1].reasoning, true);+ dom.window.close();+});diff --git a/plugins/admin/ui/settings-providers.ts b/plugins/admin/ui/settings-providers.tsindex 4a3f55120..338a121e8 100644--- a/plugins/admin/ui/settings-providers.ts+++ b/plugins/admin/ui/settings-providers.ts@@ -39,9 +39,10 @@ export function openProvider(item: any = null) {protocol: item?.protocol || "openai",url: item?.baseUrl || "",key: "",+ modelSpecs: item?.models || [],models: item? item.models- .map((m: any) => [m.id, m.name, m.contextWindow, m.maxTokens].filter((x) => x != null).join(" | "))+ .map((m: any) => [m.id, m.name, m.contextWindow, m.maxTokens, m.reasoning].map((x) => x ?? "").join(" | ")).join("\n"): "",validate: true,@@ -69,12 +70,15 @@ export function providerBody() {.map((l: string) => l.trim()).filter(Boolean).map((l: string) => {- const [id, name, contextWindow, maxTokens] = l.split("|").map((p) => p.trim());+ const [id, name, contextWindow, maxTokens, reasoning] = l.split("|").map((p) => p.trim());+ const capability = reasoning === "true" || reasoning === "false" ? reasoning === "true" : reasoning || undefined;return {+ ...d.modelSpecs?.find((model: any) => model.id === id),id,- ...(name ? { name } : {}),- ...(contextWindow ? { contextWindow: Number(contextWindow) } : {}),- ...(maxTokens ? { maxTokens: Number(maxTokens) } : {}),+ name: name || undefined,+ contextWindow: contextWindow ? Number(contextWindow) : undefined,+ maxTokens: maxTokens ? Number(maxTokens) : undefined,+ reasoning: capability,};});return {@@ -257,13 +261,14 @@ function template() {placeholder="Write-only; blank on edit keeps the stored key"/></label><label class="custom-provider-models"- >Models <span class="label-detail">one per line: id | name | context | max tokens</span+ >Models+ <span class="label-detail">one per line: id | name | context | max tokens | reasoning (true/false)</span><textareaid="custom-provider-models".value=${provider.draft.models || ""}@input=${(e: Event) => changeProvider("models", (e.target as HTMLInputElement).value)}rows="4"- placeholder="deepseek-chat | DeepSeek V3.2 | 128000 | 8192"+ placeholder="deepseek-chat | DeepSeek V3.2 | 128000 | 8192 | true"></textarea></label><label class="inline custom-provider-validation"diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.tsindex 59e0f7dc0..59fa5ddfb 100644--- a/src/harness/opencode-harness.ts+++ b/src/harness/opencode-harness.ts@@ -718,6 +718,7 @@ export function createOpenCodeHarness(opts: OpenCodeHarnessOptions = {}): Harnesm.id,{name: m.name ?? m.id,+ reasoning: m.reasoning ?? false,...(m.contextWindow || m.maxTokens? {limit: {diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.tsindex 758a8ce27..0ad3ec340 100644--- a/src/model/custom-providers.ts+++ b/src/model/custom-providers.ts@@ -22,6 +22,7 @@ export type CustomProviderProtocol = (typeof CUSTOM_PROVIDER_PROTOCOLS)[number];interface CustomModelSpec {id: string;name?: string;+ reasoning?: boolean;contextWindow?: number;maxTokens?: number;/** USD per million input tokens. Defaults to 0 (unknown / not metered). */@@ -62,6 +63,8 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {if (!m.id?.trim() || m.id.length > 200) throw new Error("every model needs an id (<=200 chars)");if (m.name !== undefined && (typeof m.name !== "string" || m.name.length > 200))throw new Error(`model "${m.id}": name must be a string of 200 chars or fewer`);+ if (m.reasoning !== undefined && typeof m.reasoning !== "boolean")+ throw new Error(`model "${m.id}": reasoning must be a boolean`);if (modelIdReserved(m.id) || (registry.has(m.id) && registry.get(m.id)?.provider !== spec.id))throw new Error(`model id "${m.id}" is already registered`);if (seen.has(m.id)) throw new Error(`duplicate model id "${m.id}"`);@@ -119,7 +122,7 @@ function toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): Custoprovider: provider.id,api: customProviderApi(provider.protocol),baseUrl: provider.baseUrl,- reasoning: false,+ reasoning: m.reasoning ?? false,input: ["text"],cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },contextWindow: m.contextWindow ?? DEFAULT_CONTEXT_WINDOW,@@ -188,6 +191,7 @@ export function customModelsJson(): { providers: Record<string, unknown> } | undmodels: spec.models.map((m) => ({id: m.id,name: m.name ?? m.id,+ reasoning: m.reasoning ?? false,contextWindow: m.contextWindow ?? 128_000,maxTokens: m.maxTokens ?? 8_192,cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },diff --git a/test/custom-provider-e2e.test.ts b/test/custom-provider-e2e.test.tsindex 25884aa0b..84337aeea 100644--- a/test/custom-provider-e2e.test.ts+++ b/test/custom-provider-e2e.test.ts@@ -128,7 +128,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()protocol: "openai",baseUrl: upstreamUrl,apiKey: "sk-qa-good",- models: [{ id: "qa-chat", name: "QA Chat", contextWindow: 64000, maxTokens: 4096 }],+ models: [{ id: "qa-chat", name: "QA Chat", contextWindow: 64000, maxTokens: 4096, reasoning: true }],}),});assert.equal(r.status, 200);@@ -147,6 +147,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()const model = resolveModel("qa-chat");assert.ok(model, "custom model resolves");assert.equal(model!.provider, "qa");+ assert.equal(model!.reasoning, true);assert.equal((model as { baseUrl?: string }).baseUrl, upstreamUrl);assert.equal(modelSupportedByHarness("qa-chat", "pi"), true);assert.equal(modelSupportedByHarness("qa-chat", "opencode"), true);diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.tsindex afcbe264b..c02c5c642 100644--- a/test/custom-providers.test.ts+++ b/test/custom-providers.test.ts@@ -224,3 +224,33 @@ for (const field of ["contextWindow", "maxTokens"] as const) {);});}++for (const protocol of ["openai", "openai-responses", "anthropic"] as const) {+ for (const reasoning of [undefined, false, true]) {+ test(`custom ${protocol} reasoning=${reasoning} survives persistence and runtime mappings`, async () => {+ const spec = { ...GATEWAY, protocol, models: [{ ...GATEWAY.models[0]!, reasoning }] };+ const backing = createMemoryMap<StoredCustomProvider>();+ const store = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });+ await store.upsert(spec, "local", "[email protected]");+ const reloaded = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });+ setCustomProviders(await reloaded.enabled());+ const model = resolveCustomModel("acme-large")!;+ assert.equal(model.reasoning, reasoning ?? false);+ const json = customModelsJson() as {+ providers: Record<string, { models: Array<{ reasoning: boolean; cost: typeof model.cost }> }>;+ };+ assert.equal(json.providers[spec.id]!.models[0]!.reasoning, reasoning ?? false);+ assert.deepEqual(json.providers[spec.id]!.models[0]!.cost, model.cost);+ assert.deepEqual((await reloaded.statuses())[0]!.models, spec.models);+ });+ }+}++test("custom provider reasoning rejects non-boolean capabilities", () => {+ for (const reasoning of ["true", "false", 0, 1, null, {}]) {+ assert.throws(+ () => validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", reasoning } as never] }),+ /reasoning must be a boolean/,+ );+ }+});diff --git a/test/opencode-harness.test.ts b/test/opencode-harness.test.tsindex 8d8c74000..ec15aa6ea 100644--- a/test/opencode-harness.test.ts+++ b/test/opencode-harness.test.ts@@ -378,7 +378,7 @@ test("custom providers materialize into the opencode config (enabled + providername: "LiteLLM",protocol: "openai" as const,baseUrl: "http://litellm.internal:4000/v1",- models: [{ id: "deepseek-chat", name: "DeepSeek", contextWindow: 128000, maxTokens: 8192 }],+ models: [{ id: "deepseek-chat", name: "DeepSeek", contextWindow: 128000, maxTokens: 8192, reasoning: true }],},apiKey: "sk-lite",},@@ -405,7 +405,12 @@ test("custom providers materialize into the opencode config (enabled + providerassert.equal(litellm.npm, "@ai-sdk/openai-compatible");assert.equal(litellm.options.baseURL, "http://litellm.internal:4000/v1");assert.equal(litellm.options.apiKey, "sk-lite");- assert.deepEqual(litellm.models["deepseek-chat"], { name: "DeepSeek", limit: { context: 128000, output: 8192 } });+ assert.deepEqual(litellm.models["deepseek-chat"], {+ name: "DeepSeek",+ reasoning: true,+ limit: { context: 128000, output: 8192 },+ });+ assert.equal(config.provider["responses-proxy"].models["responses-model"].reasoning, false);assert.equal(config.provider["responses-proxy"].npm, "@ai-sdk/openai");assert.equal(config.provider["responses-proxy"].options.apiKey, "sk-responses");} finally {