Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Give custom Anthropic models the thinking binding of the known model they name (#2528)

Joshua Francee3a1232151ae
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index b7d4d2507..dc821290e 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -77,9 +77,15 @@ import {
codexProviderModelId,
modelHasPrices,
} from "../model/pi-models.ts";
-import { customModelsJson, customProvidersVersion, resolveCustomModel } from "../model/custom-providers.ts";
+import {
+ customModelsJson,
+ customProvidersVersion,
+ hasCustomModelPrefix,
+ resolveCustomModel,
+} from "../model/custom-providers.ts";
import {
GatewayModelUnavailableError,
+ ModelUnavailableError,
modelGatewayRequest,
type ModelGatewayTransportConfig,
} from "../model/provider-endpoints.ts";
@@ -1225,10 +1231,12 @@ export async function buildModelRuntime(
): { model: Model<Api>; options: T } => {
const request = modelGatewayRequest(modelGateway, model);
if (!request) {
+ const custom = resolveCustomModel(model.id);
const providerModelId =
model.provider === CODEX_SUBSCRIPTION_PROVIDER
? codexProviderModelId(model.id)
- : (modelRequestOverrides(model.id)?.model ?? resolveCustomModel(model.id)?.upstreamId ?? model.id);
+ : (modelRequestOverrides(model.id)?.model ?? custom?.upstreamId ?? model.id);
+ const unregistered = !custom && hasCustomModelPrefix(model.id);
const candidate = withRequestHeaders(model, true, false);
const passthrough = {
...retained(options),
@@ -1240,7 +1248,12 @@ export async function buildModelRuntime(
return {
model: candidate,
options:
- providerModelId === model.id ? passthrough : wireModelId(passthrough, model, async () => providerModelId),
+ providerModelId === model.id && !unregistered
+ ? passthrough
+ : wireModelId(passthrough, model, async () => {
+ if (unregistered) throw new ModelUnavailableError(model.id);
+ return providerModelId;
+ }),
};
}
const routed = {
diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.ts
index 9dc63e9b0..7f3af2518 100644
--- a/src/model/custom-providers.ts
+++ b/src/model/custom-providers.ts
@@ -104,6 +104,7 @@ export interface CustomRuntimeModel {
maxTokens: number;
}
+const CUSTOM_MODEL_PREFIX = "custom/";
const DEFAULT_CONTEXT_WINDOW = 128_000;
const DEFAULT_MAX_TOKENS = 8_192;
@@ -115,7 +116,7 @@ function customProviderApi(protocol: CustomProviderProtocol): CustomRuntimeModel
function toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): CustomRuntimeModel {
return {
- id: `custom/${provider.id}/${m.id}`,
+ id: `${CUSTOM_MODEL_PREFIX}${provider.id}/${m.id}`,
upstreamId: m.id,
name: m.name?.trim() || m.id,
provider: provider.id,
@@ -159,6 +160,10 @@ export function customProvidersVersion(): number {
return version;
}
+export function hasCustomModelPrefix(id: string): boolean {
+ return id.startsWith(CUSTOM_MODEL_PREFIX);
+}
+
export function resolveCustomModel(id: string): CustomRuntimeModel | undefined {
return registry.get(id);
}
diff --git a/src/model/model-verification.ts b/src/model/model-verification.ts
index 582f1d183..e2a148478 100644
--- a/src/model/model-verification.ts
+++ b/src/model/model-verification.ts
@@ -6,7 +6,7 @@ import { embeddingModelFromDefinition, modelFromOverlay } from "./pi-models.ts";
import { isEmbeddingModel, type RegisteredModel } from "./model-overlay.ts";
import { createMemoryEmbedder } from "./embeddings.ts";
import type { ModelCredentialStore } from "./model-credential-store.ts";
-import { GatewayModelUnavailableError, type ModelGatewayTransportConfig } from "./provider-endpoints.ts";
+import { ModelUnavailableError, type ModelGatewayTransportConfig } from "./provider-endpoints.ts";
export class ModelVerificationError extends Error {
readonly code: string;
@@ -49,7 +49,7 @@ export function verificationFailure(error: unknown): ModelVerificationError {
if (error instanceof ModelVerificationError) return error;
let code = "provider_failure";
if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) code = "timeout";
- else if (error instanceof GatewayModelUnavailableError) code = "model_unavailable";
+ else if (error instanceof ModelUnavailableError) code = "model_unavailable";
else if (error instanceof ProviderTurnError) code = VERIFICATION_CODES[error.code];
reportFailure("model verification", error);
return new ModelVerificationError(code, VERIFICATION_FAILURES[code]!, { cause: error });
diff --git a/src/model/pi-models.ts b/src/model/pi-models.ts
index 95b466755..649230be4 100644
--- a/src/model/pi-models.ts
+++ b/src/model/pi-models.ts
@@ -17,7 +17,7 @@ import {
type ModelOverlay,
} from "./model-overlay.ts";
import { providerBaseUrl } from "./provider-endpoints.ts";
-import { isCustomModelId, resolveCustomModel } from "./custom-providers.ts";
+import { type CustomRuntimeModel, isCustomModelId, resolveCustomModel } from "./custom-providers.ts";
const getModel = getBuiltinModel as unknown as (provider: string, id: string) => Model<Api> | undefined;
@@ -631,7 +631,20 @@ function resolveBaseModel(id: string): PiModel | undefined {
if (overlay && !isEmbeddingModel(overlay)) {
return modelFromOverlay(overlay, false);
}
- return (resolveCustomModel(id) as unknown as PiModel | undefined) ?? OPENROUTER_CATALOG_MODELS.get(id);
+ const custom = resolveCustomModel(id);
+ return custom ? customPiModel(custom) : OPENROUTER_CATALOG_MODELS.get(id);
+}
+
+function customPiModel(custom: CustomRuntimeModel): PiModel {
+ const model = custom as unknown as PiModel;
+ if (model.api !== "anthropic-messages") return model;
+ const known = resolveBuiltinModel(custom.upstreamId);
+ if (known?.api !== "anthropic-messages" || !modelSupportsAdaptiveThinking(known)) return model;
+ return {
+ ...model,
+ compat: { forceAdaptiveThinking: true },
+ ...(known.thinkingLevelMap ? { thinkingLevelMap: known.thinkingLevelMap } : {}),
+ };
}
export function modelFromOverlay(spec: ModelOverlay, useOrgEndpoints = true): PiModel | undefined {
diff --git a/src/model/provider-endpoints.ts b/src/model/provider-endpoints.ts
index 73e42a264..b8dc2d866 100644
--- a/src/model/provider-endpoints.ts
+++ b/src/model/provider-endpoints.ts
@@ -74,14 +74,18 @@ export function providerBaseUrl(provider: string): string | undefined {
return (PROVIDER_IDS as readonly string[]).includes(provider) ? configured[provider as ProviderId] : undefined;
}
-/** Our own signal that the gateway has no route for a model id; typed so callers never match its text. */
-export class GatewayModelUnavailableError extends Error {
- // Read by the vendored pi-ai's extractProviderError when this is thrown mid-stream, so the failed
- // message carries providerError {status: 404, type: "model_unavailable"}.
+export class ModelUnavailableError extends Error {
readonly status = 404;
readonly type = "model_unavailable";
+ constructor(modelId: string, label = "Model") {
+ super(`${label} is unavailable: ${modelId}`);
+ this.name = "ModelUnavailableError";
+ }
+}
+
+export class GatewayModelUnavailableError extends ModelUnavailableError {
constructor(modelId: string) {
- super(`Gateway model is unavailable: ${modelId}`);
+ super(modelId, "Gateway model");
this.name = "GatewayModelUnavailableError";
}
}
diff --git a/test/custom-anthropic-thinking.test.ts b/test/custom-anthropic-thinking.test.ts
new file mode 100644
index 000000000..6a9247d25
--- /dev/null
+++ b/test/custom-anthropic-thinking.test.ts
@@ -0,0 +1,169 @@
+import assert from "node:assert/strict";
+import { createServer, type IncomingHttpHeaders } from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterEach, test } from "node:test";
+import type { Api, AssistantMessage, Context, Model } from "@earendil-works/pi-ai";
+import { buildModelRuntime } from "../src/harness/pi-harness.ts";
+import { resolveModel } from "../src/model/pi-models.ts";
+import { setCustomProviders } from "../src/model/custom-providers.ts";
+
+const BINDING_BETA = "thinking-binding-controls-2026-08-01";
+const DROP_BLOCK = { prefix_mismatch_behavior: "drop_block" };
+
+afterEach(() => setCustomProviders([]));
+
+type Seen = { headers: IncomingHttpHeaders; body: Record<string, unknown> };
+
+async function fakeAnthropic(): Promise<{ url: string; seen: Seen[]; close: () => void }> {
+ const seen: Seen[] = [];
+ const server = createServer((req, res) => {
+ let raw = "";
+ req.on("data", (chunk) => (raw += chunk));
+ req.on("end", () => {
+ seen.push({ headers: req.headers, body: JSON.parse(raw) as Record<string, unknown> });
+ const event = (name: string, data: object) =>
+ `event: ${name}\ndata: ${JSON.stringify({ type: name, ...data })}\n\n`;
+ res.writeHead(200, { "content-type": "text/event-stream" });
+ res.write(
+ event("message_start", {
+ message: {
+ id: "msg_1",
+ type: "message",
+ role: "assistant",
+ content: [],
+ model: "m",
+ stop_reason: null,
+ usage: { input_tokens: 1, output_tokens: 0 },
+ },
+ }),
+ );
+ res.write(event("content_block_start", { index: 0, content_block: { type: "text", text: "" } }));
+ res.write(event("content_block_delta", { index: 0, delta: { type: "text_delta", text: "OK" } }));
+ res.write(event("content_block_stop", { index: 0 }));
+ res.write(event("message_delta", { delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }));
+ res.write(event("message_stop", {}));
+ res.end();
+ });
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ return {
+ url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
+ seen,
+ close: () => server.close(),
+ };
+}
+
+function register(baseUrl: string, models: Array<{ id: string; reasoning?: boolean }>, protocol = "anthropic"): void {
+ setCustomProviders([
+ { id: "direct", name: "Direct", protocol: protocol as "anthropic", baseUrl, models: models.map((m) => ({ ...m })) },
+ ]);
+}
+
+function replayContext(model: Model<Api>, firstPrompt: string): Context {
+ const prior: AssistantMessage = {
+ role: "assistant",
+ content: [
+ { type: "thinking", thinking: "working it out", thinkingSignature: "signed-block" },
+ { type: "text", text: "391" },
+ ],
+ api: model.api,
+ provider: model.provider,
+ model: model.id,
+ usage: {
+ input: 1,
+ output: 1,
+ cacheRead: 0,
+ cacheWrite: 0,
+ totalTokens: 2,
+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+ },
+ stopReason: "stop",
+ timestamp: Date.now(),
+ };
+ return {
+ systemPrompt: "terse",
+ messages: [
+ { role: "user", content: firstPrompt, timestamp: Date.now() },
+ prior,
+ { role: "user", content: "add 9", timestamp: Date.now() },
+ ],
+ };
+}
+
+async function send(model: Model<Api>, reasoning?: "high"): Promise<void> {
+ const runtime = await buildModelRuntime({ direct: "sk-direct" });
+ const reply = await runtime
+ .streamSimple(model, replayContext(model, "rewritten context: what is 17*23?"), reasoning ? { reasoning } : {})
+ .result();
+ assert.equal(reply.stopReason, "stop", reply.errorMessage);
+}
+
+test("a custom anthropic model of a known adaptive model sends thinking binding even with reasoning off", async () => {
+ const upstream = await fakeAnthropic();
+ try {
+ register(upstream.url, [{ id: "claude-opus-5-5" }]);
+ const model = resolveModel("custom/direct/claude-opus-5-5") as Model<Api>;
+ await send(model);
+ const [call] = upstream.seen;
+ assert.ok(call);
+ assert.equal(call.body.model, "claude-opus-5-5");
+ assert.ok(String(call.headers["anthropic-beta"]).split(",").includes(BINDING_BETA));
+ assert.deepEqual(call.body.thinking, { type: "adaptive", display: "summarized", block_binding: DROP_BLOCK });
+ const replayed = (call.body.messages as Array<{ content: unknown }>)[1]!.content as Array<{ type: string }>;
+ assert.equal(replayed[0]!.type, "thinking");
+ } finally {
+ upstream.close();
+ }
+});
+
+test("reasoning on a custom anthropic model of a known adaptive model uses adaptive thinking with binding", async () => {
+ const upstream = await fakeAnthropic();
+ try {
+ register(upstream.url, [{ id: "claude-opus-5-5", reasoning: true }]);
+ await send(resolveModel("custom/direct/claude-opus-5-5") as Model<Api>, "high");
+ const thinking = upstream.seen[0]!.body.thinking as Record<string, unknown>;
+ assert.equal(thinking.type, "adaptive");
+ assert.deepEqual(thinking.block_binding, DROP_BLOCK);
+ assert.equal(thinking.budget_tokens, undefined);
+ } finally {
+ upstream.close();
+ }
+});
+
+test("a custom anthropic model with an unrecognized id keeps the plain request", async () => {
+ const upstream = await fakeAnthropic();
+ try {
+ register(upstream.url, [{ id: "compat-large" }]);
+ await send(resolveModel("custom/direct/compat-large") as Model<Api>);
+ const [call] = upstream.seen;
+ assert.equal(call!.headers["anthropic-beta"]?.includes(BINDING_BETA) ?? false, false);
+ assert.equal(call!.body.thinking, undefined);
+ } finally {
+ upstream.close();
+ }
+});
+
+test("only anthropic-protocol custom models take on a known model's thinking traits", () => {
+ register("https://llm.example.test/v1", [{ id: "claude-opus-5-5" }], "openai");
+ const model = resolveModel("custom/direct/claude-opus-5-5");
+ assert.equal(model?.api, "openai-completions");
+ assert.equal(model?.compat, undefined);
+ assert.equal(model?.thinkingLevelMap, undefined);
+});
+
+test("a custom model whose provider is gone fails as unavailable instead of sending its id upstream", async () => {
+ const upstream = await fakeAnthropic();
+ try {
+ register(upstream.url, [{ id: "claude-opus-5-5" }]);
+ const model = resolveModel("custom/direct/claude-opus-5-5") as Model<Api>;
+ const runtime = await buildModelRuntime({ direct: "sk-direct" });
+ setCustomProviders([]);
+ const reply = await runtime.streamSimple(model, replayContext(model, "hi"), {}).result();
+ assert.equal(reply.stopReason, "error");
+ assert.match(String(reply.errorMessage), /Model is unavailable: custom\/direct\/claude-opus-5-5/);
+ assert.equal((reply as { providerError?: { type?: string } }).providerError?.type, "model_unavailable");
+ assert.equal(upstream.seen.length, 0);
+ } finally {
+ upstream.close();
+ }
+});