Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Allow reasoning capability on custom models (#1810)

Sina Matian9627efc71109
diff --git a/plugins/admin/test/settings-provider-races.test.ts b/plugins/admin/test/settings-provider-races.test.ts
index cc41d4941..c3d982ea0 100644
--- a/plugins/admin/test/settings-provider-races.test.ts
+++ b/plugins/admin/test/settings-provider-races.test.ts
@@ -78,3 +78,28 @@ test("old provider save cannot clear pending state in a reopened dialog", async
assert.equal((dom.window.document.getElementById("custom-provider-dialog") as HTMLDialogElement).open, false);
dom.window.close();
});
+
+test("reasoning edits preserve pricing and omitted positional fields", () => {
+ const { dom, ui } = fixture();
+ ui.openProvider({
+ id: "local",
+ name: "Local",
+ protocol: "openai",
+ baseUrl: "http://localhost:8000/v1",
+ models: [
+ { id: "thinking", contextWindow: 32768, maxTokens: 4096, reasoning: true, input: 0, output: 2 },
+ { id: "plain", name: "Plain", reasoning: false },
+ ],
+ });
+ assert.equal(ui.provider.draft.models, "thinking | | 32768 | 4096 | true\nplain | Plain | | | false");
+ assert.deepEqual(JSON.parse(JSON.stringify(ui.providerBody().models)), [
+ { id: "thinking", contextWindow: 32768, maxTokens: 4096, reasoning: true, input: 0, output: 2 },
+ { id: "plain", name: "Plain", reasoning: false },
+ ]);
+ ui.provider.draft.models = "thinking | | 32768 | 4096 | false\nnew-model | New | 16000 | 2000 | true";
+ const models = ui.providerBody().models;
+ assert.equal(models[0].reasoning, false);
+ assert.equal(models[0].output, 2);
+ assert.equal(models[1].reasoning, true);
+ dom.window.close();
+});
diff --git a/plugins/admin/ui/settings-providers.ts b/plugins/admin/ui/settings-providers.ts
index 4a3f55120..338a121e8 100644
--- a/plugins/admin/ui/settings-providers.ts
+++ b/plugins/admin/ui/settings-providers.ts
@@ -39,9 +39,10 @@ export function openProvider(item: any = null) {
protocol: item?.protocol || "openai",
url: item?.baseUrl || "",
key: "",
+ modelSpecs: item?.models || [],
models: item
? item.models
- .map((m: any) => [m.id, m.name, m.contextWindow, m.maxTokens].filter((x) => x != null).join(" | "))
+ .map((m: any) => [m.id, m.name, m.contextWindow, m.maxTokens, m.reasoning].map((x) => x ?? "").join(" | "))
.join("\n")
: "",
validate: true,
@@ -69,12 +70,15 @@ export function providerBody() {
.map((l: string) => l.trim())
.filter(Boolean)
.map((l: string) => {
- const [id, name, contextWindow, maxTokens] = l.split("|").map((p) => p.trim());
+ const [id, name, contextWindow, maxTokens, reasoning] = l.split("|").map((p) => p.trim());
+ const capability = reasoning === "true" || reasoning === "false" ? reasoning === "true" : reasoning || undefined;
return {
+ ...d.modelSpecs?.find((model: any) => model.id === id),
id,
- ...(name ? { name } : {}),
- ...(contextWindow ? { contextWindow: Number(contextWindow) } : {}),
- ...(maxTokens ? { maxTokens: Number(maxTokens) } : {}),
+ name: name || undefined,
+ contextWindow: contextWindow ? Number(contextWindow) : undefined,
+ maxTokens: maxTokens ? Number(maxTokens) : undefined,
+ reasoning: capability,
};
});
return {
@@ -257,13 +261,14 @@ function template() {
placeholder="Write-only; blank on edit keeps the stored key"
/></label>
<label class="custom-provider-models"
- >Models <span class="label-detail">one per line: id | name | context | max tokens</span
+ >Models
+ <span class="label-detail">one per line: id | name | context | max tokens | reasoning (true/false)</span
><textarea
id="custom-provider-models"
.value=${provider.draft.models || ""}
@input=${(e: Event) => changeProvider("models", (e.target as HTMLInputElement).value)}
rows="4"
- placeholder="deepseek-chat | DeepSeek V3.2 | 128000 | 8192"
+ placeholder="deepseek-chat | DeepSeek V3.2 | 128000 | 8192 | true"
></textarea>
</label>
<label class="inline custom-provider-validation"
diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.ts
index 59e0f7dc0..59fa5ddfb 100644
--- a/src/harness/opencode-harness.ts
+++ b/src/harness/opencode-harness.ts
@@ -718,6 +718,7 @@ export function createOpenCodeHarness(opts: OpenCodeHarnessOptions = {}): Harnes
m.id,
{
name: m.name ?? m.id,
+ reasoning: m.reasoning ?? false,
...(m.contextWindow || m.maxTokens
? {
limit: {
diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.ts
index 758a8ce27..0ad3ec340 100644
--- a/src/model/custom-providers.ts
+++ b/src/model/custom-providers.ts
@@ -22,6 +22,7 @@ export type CustomProviderProtocol = (typeof CUSTOM_PROVIDER_PROTOCOLS)[number];
interface CustomModelSpec {
id: string;
name?: string;
+ reasoning?: boolean;
contextWindow?: number;
maxTokens?: number;
/** USD per million input tokens. Defaults to 0 (unknown / not metered). */
@@ -62,6 +63,8 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {
if (!m.id?.trim() || m.id.length > 200) throw new Error("every model needs an id (<=200 chars)");
if (m.name !== undefined && (typeof m.name !== "string" || m.name.length > 200))
throw new Error(`model "${m.id}": name must be a string of 200 chars or fewer`);
+ if (m.reasoning !== undefined && typeof m.reasoning !== "boolean")
+ throw new Error(`model "${m.id}": reasoning must be a boolean`);
if (modelIdReserved(m.id) || (registry.has(m.id) && registry.get(m.id)?.provider !== spec.id))
throw new Error(`model id "${m.id}" is already registered`);
if (seen.has(m.id)) throw new Error(`duplicate model id "${m.id}"`);
@@ -119,7 +122,7 @@ function toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): Custo
provider: provider.id,
api: customProviderApi(provider.protocol),
baseUrl: provider.baseUrl,
- reasoning: false,
+ reasoning: m.reasoning ?? false,
input: ["text"],
cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: m.contextWindow ?? DEFAULT_CONTEXT_WINDOW,
@@ -188,6 +191,7 @@ export function customModelsJson(): { providers: Record<string, unknown> } | und
models: spec.models.map((m) => ({
id: m.id,
name: m.name ?? m.id,
+ reasoning: m.reasoning ?? false,
contextWindow: m.contextWindow ?? 128_000,
maxTokens: m.maxTokens ?? 8_192,
cost: { input: m.input ?? 0, output: m.output ?? 0, cacheRead: 0, cacheWrite: 0 },
diff --git a/test/custom-provider-e2e.test.ts b/test/custom-provider-e2e.test.ts
index 25884aa0b..84337aeea 100644
--- a/test/custom-provider-e2e.test.ts
+++ b/test/custom-provider-e2e.test.ts
@@ -128,7 +128,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()
protocol: "openai",
baseUrl: upstreamUrl,
apiKey: "sk-qa-good",
- models: [{ id: "qa-chat", name: "QA Chat", contextWindow: 64000, maxTokens: 4096 }],
+ models: [{ id: "qa-chat", name: "QA Chat", contextWindow: 64000, maxTokens: 4096, reasoning: true }],
}),
});
assert.equal(r.status, 200);
@@ -147,6 +147,7 @@ test("QA: full custom-provider lifecycle against a live fake upstream", async ()
const model = resolveModel("qa-chat");
assert.ok(model, "custom model resolves");
assert.equal(model!.provider, "qa");
+ assert.equal(model!.reasoning, true);
assert.equal((model as { baseUrl?: string }).baseUrl, upstreamUrl);
assert.equal(modelSupportedByHarness("qa-chat", "pi"), true);
assert.equal(modelSupportedByHarness("qa-chat", "opencode"), true);
diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.ts
index afcbe264b..c02c5c642 100644
--- a/test/custom-providers.test.ts
+++ b/test/custom-providers.test.ts
@@ -224,3 +224,33 @@ for (const field of ["contextWindow", "maxTokens"] as const) {
);
});
}
+
+for (const protocol of ["openai", "openai-responses", "anthropic"] as const) {
+ for (const reasoning of [undefined, false, true]) {
+ test(`custom ${protocol} reasoning=${reasoning} survives persistence and runtime mappings`, async () => {
+ const spec = { ...GATEWAY, protocol, models: [{ ...GATEWAY.models[0]!, reasoning }] };
+ const backing = createMemoryMap<StoredCustomProvider>();
+ const store = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });
+ await store.upsert(spec, "local", "[email protected]");
+ const reloaded = createCustomProviderStore({ backing, keyMaterial: "reasoning-test" });
+ setCustomProviders(await reloaded.enabled());
+ const model = resolveCustomModel("acme-large")!;
+ assert.equal(model.reasoning, reasoning ?? false);
+ const json = customModelsJson() as {
+ providers: Record<string, { models: Array<{ reasoning: boolean; cost: typeof model.cost }> }>;
+ };
+ assert.equal(json.providers[spec.id]!.models[0]!.reasoning, reasoning ?? false);
+ assert.deepEqual(json.providers[spec.id]!.models[0]!.cost, model.cost);
+ assert.deepEqual((await reloaded.statuses())[0]!.models, spec.models);
+ });
+ }
+}
+
+test("custom provider reasoning rejects non-boolean capabilities", () => {
+ for (const reasoning of ["true", "false", 0, 1, null, {}]) {
+ assert.throws(
+ () => validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", reasoning } as never] }),
+ /reasoning must be a boolean/,
+ );
+ }
+});
diff --git a/test/opencode-harness.test.ts b/test/opencode-harness.test.ts
index 8d8c74000..ec15aa6ea 100644
--- a/test/opencode-harness.test.ts
+++ b/test/opencode-harness.test.ts
@@ -378,7 +378,7 @@ test("custom providers materialize into the opencode config (enabled + provider
name: "LiteLLM",
protocol: "openai" as const,
baseUrl: "http://litellm.internal:4000/v1",
- models: [{ id: "deepseek-chat", name: "DeepSeek", contextWindow: 128000, maxTokens: 8192 }],
+ models: [{ id: "deepseek-chat", name: "DeepSeek", contextWindow: 128000, maxTokens: 8192, reasoning: true }],
},
apiKey: "sk-lite",
},
@@ -405,7 +405,12 @@ test("custom providers materialize into the opencode config (enabled + provider
assert.equal(litellm.npm, "@ai-sdk/openai-compatible");
assert.equal(litellm.options.baseURL, "http://litellm.internal:4000/v1");
assert.equal(litellm.options.apiKey, "sk-lite");
- assert.deepEqual(litellm.models["deepseek-chat"], { name: "DeepSeek", limit: { context: 128000, output: 8192 } });
+ assert.deepEqual(litellm.models["deepseek-chat"], {
+ name: "DeepSeek",
+ reasoning: true,
+ limit: { context: 128000, output: 8192 },
+ });
+ assert.equal(config.provider["responses-proxy"].models["responses-model"].reasoning, false);
assert.equal(config.provider["responses-proxy"].npm, "@ai-sdk/openai");
assert.equal(config.provider["responses-proxy"].options.apiKey, "sk-responses");
} finally {