Multiplayer agent harness for work.
Give custom Anthropic models the thinking binding of the known model they name (#2528)
Joshua Francee3a1232151ae
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex b7d4d2507..dc821290e 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -77,9 +77,15 @@ import {codexProviderModelId,modelHasPrices,} from "../model/pi-models.ts";-import { customModelsJson, customProvidersVersion, resolveCustomModel } from "../model/custom-providers.ts";+import {+ customModelsJson,+ customProvidersVersion,+ hasCustomModelPrefix,+ resolveCustomModel,+} from "../model/custom-providers.ts";import {GatewayModelUnavailableError,+ ModelUnavailableError,modelGatewayRequest,type ModelGatewayTransportConfig,} from "../model/provider-endpoints.ts";@@ -1225,10 +1231,12 @@ export async function buildModelRuntime(): { model: Model<Api>; options: T } => {const request = modelGatewayRequest(modelGateway, model);if (!request) {+ const custom = resolveCustomModel(model.id);const providerModelId =model.provider === CODEX_SUBSCRIPTION_PROVIDER? codexProviderModelId(model.id)- : (modelRequestOverrides(model.id)?.model ?? resolveCustomModel(model.id)?.upstreamId ?? model.id);+ : (modelRequestOverrides(model.id)?.model ?? custom?.upstreamId ?? model.id);+ const unregistered = !custom && hasCustomModelPrefix(model.id);const candidate = withRequestHeaders(model, true, false);const passthrough = {...retained(options),@@ -1240,7 +1248,12 @@ export async function buildModelRuntime(return {model: candidate,options:- providerModelId === model.id ? passthrough : wireModelId(passthrough, model, async () => providerModelId),+ providerModelId === model.id && !unregistered+ ? passthrough+ : wireModelId(passthrough, model, async () => {+ if (unregistered) throw new ModelUnavailableError(model.id);+ return providerModelId;+ }),};}const routed = {diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.tsindex 9dc63e9b0..7f3af2518 100644--- a/src/model/custom-providers.ts+++ b/src/model/custom-providers.ts@@ -104,6 +104,7 @@ export interface CustomRuntimeModel {maxTokens: number;}+const CUSTOM_MODEL_PREFIX = "custom/";const DEFAULT_CONTEXT_WINDOW = 128_000;const DEFAULT_MAX_TOKENS = 8_192;@@ -115,7 +116,7 @@ function customProviderApi(protocol: CustomProviderProtocol): CustomRuntimeModelfunction toRuntimeModel(provider: CustomProviderSpec, m: CustomModelSpec): CustomRuntimeModel {return {- id: `custom/${provider.id}/${m.id}`,+ id: `${CUSTOM_MODEL_PREFIX}${provider.id}/${m.id}`,upstreamId: m.id,name: m.name?.trim() || m.id,provider: provider.id,@@ -159,6 +160,10 @@ export function customProvidersVersion(): number {return version;}+export function hasCustomModelPrefix(id: string): boolean {+ return id.startsWith(CUSTOM_MODEL_PREFIX);+}+export function resolveCustomModel(id: string): CustomRuntimeModel | undefined {return registry.get(id);}diff --git a/src/model/model-verification.ts b/src/model/model-verification.tsindex 582f1d183..e2a148478 100644--- a/src/model/model-verification.ts+++ b/src/model/model-verification.ts@@ -6,7 +6,7 @@ import { embeddingModelFromDefinition, modelFromOverlay } from "./pi-models.ts";import { isEmbeddingModel, type RegisteredModel } from "./model-overlay.ts";import { createMemoryEmbedder } from "./embeddings.ts";import type { ModelCredentialStore } from "./model-credential-store.ts";-import { GatewayModelUnavailableError, type ModelGatewayTransportConfig } from "./provider-endpoints.ts";+import { ModelUnavailableError, type ModelGatewayTransportConfig } from "./provider-endpoints.ts";export class ModelVerificationError extends Error {readonly code: string;@@ -49,7 +49,7 @@ export function verificationFailure(error: unknown): ModelVerificationError {if (error instanceof ModelVerificationError) return error;let code = "provider_failure";if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) code = "timeout";- else if (error instanceof GatewayModelUnavailableError) code = "model_unavailable";+ else if (error instanceof ModelUnavailableError) code = "model_unavailable";else if (error instanceof ProviderTurnError) code = VERIFICATION_CODES[error.code];reportFailure("model verification", error);return new ModelVerificationError(code, VERIFICATION_FAILURES[code]!, { cause: error });diff --git a/src/model/pi-models.ts b/src/model/pi-models.tsindex 95b466755..649230be4 100644--- a/src/model/pi-models.ts+++ b/src/model/pi-models.ts@@ -17,7 +17,7 @@ import {type ModelOverlay,} from "./model-overlay.ts";import { providerBaseUrl } from "./provider-endpoints.ts";-import { isCustomModelId, resolveCustomModel } from "./custom-providers.ts";+import { type CustomRuntimeModel, isCustomModelId, resolveCustomModel } from "./custom-providers.ts";const getModel = getBuiltinModel as unknown as (provider: string, id: string) => Model<Api> | undefined;@@ -631,7 +631,20 @@ function resolveBaseModel(id: string): PiModel | undefined {if (overlay && !isEmbeddingModel(overlay)) {return modelFromOverlay(overlay, false);}- return (resolveCustomModel(id) as unknown as PiModel | undefined) ?? OPENROUTER_CATALOG_MODELS.get(id);+ const custom = resolveCustomModel(id);+ return custom ? customPiModel(custom) : OPENROUTER_CATALOG_MODELS.get(id);+}++function customPiModel(custom: CustomRuntimeModel): PiModel {+ const model = custom as unknown as PiModel;+ if (model.api !== "anthropic-messages") return model;+ const known = resolveBuiltinModel(custom.upstreamId);+ if (known?.api !== "anthropic-messages" || !modelSupportsAdaptiveThinking(known)) return model;+ return {+ ...model,+ compat: { forceAdaptiveThinking: true },+ ...(known.thinkingLevelMap ? { thinkingLevelMap: known.thinkingLevelMap } : {}),+ };}export function modelFromOverlay(spec: ModelOverlay, useOrgEndpoints = true): PiModel | undefined {diff --git a/src/model/provider-endpoints.ts b/src/model/provider-endpoints.tsindex 73e42a264..b8dc2d866 100644--- a/src/model/provider-endpoints.ts+++ b/src/model/provider-endpoints.ts@@ -74,14 +74,18 @@ export function providerBaseUrl(provider: string): string | undefined {return (PROVIDER_IDS as readonly string[]).includes(provider) ? configured[provider as ProviderId] : undefined;}-/** Our own signal that the gateway has no route for a model id; typed so callers never match its text. */-export class GatewayModelUnavailableError extends Error {- // Read by the vendored pi-ai's extractProviderError when this is thrown mid-stream, so the failed- // message carries providerError {status: 404, type: "model_unavailable"}.+export class ModelUnavailableError extends Error {readonly status = 404;readonly type = "model_unavailable";+ constructor(modelId: string, label = "Model") {+ super(`${label} is unavailable: ${modelId}`);+ this.name = "ModelUnavailableError";+ }+}++export class GatewayModelUnavailableError extends ModelUnavailableError {constructor(modelId: string) {- super(`Gateway model is unavailable: ${modelId}`);+ super(modelId, "Gateway model");this.name = "GatewayModelUnavailableError";}}diff --git a/test/custom-anthropic-thinking.test.ts b/test/custom-anthropic-thinking.test.tsnew file mode 100644index 000000000..6a9247d25--- /dev/null+++ b/test/custom-anthropic-thinking.test.ts@@ -0,0 +1,169 @@+import assert from "node:assert/strict";+import { createServer, type IncomingHttpHeaders } from "node:http";+import type { AddressInfo } from "node:net";+import { afterEach, test } from "node:test";+import type { Api, AssistantMessage, Context, Model } from "@earendil-works/pi-ai";+import { buildModelRuntime } from "../src/harness/pi-harness.ts";+import { resolveModel } from "../src/model/pi-models.ts";+import { setCustomProviders } from "../src/model/custom-providers.ts";++const BINDING_BETA = "thinking-binding-controls-2026-08-01";+const DROP_BLOCK = { prefix_mismatch_behavior: "drop_block" };++afterEach(() => setCustomProviders([]));++type Seen = { headers: IncomingHttpHeaders; body: Record<string, unknown> };++async function fakeAnthropic(): Promise<{ url: string; seen: Seen[]; close: () => void }> {+ const seen: Seen[] = [];+ const server = createServer((req, res) => {+ let raw = "";+ req.on("data", (chunk) => (raw += chunk));+ req.on("end", () => {+ seen.push({ headers: req.headers, body: JSON.parse(raw) as Record<string, unknown> });+ const event = (name: string, data: object) =>+ `event: ${name}\ndata: ${JSON.stringify({ type: name, ...data })}\n\n`;+ res.writeHead(200, { "content-type": "text/event-stream" });+ res.write(+ event("message_start", {+ message: {+ id: "msg_1",+ type: "message",+ role: "assistant",+ content: [],+ model: "m",+ stop_reason: null,+ usage: { input_tokens: 1, output_tokens: 0 },+ },+ }),+ );+ res.write(event("content_block_start", { index: 0, content_block: { type: "text", text: "" } }));+ res.write(event("content_block_delta", { index: 0, delta: { type: "text_delta", text: "OK" } }));+ res.write(event("content_block_stop", { index: 0 }));+ res.write(event("message_delta", { delta: { stop_reason: "end_turn" }, usage: { output_tokens: 1 } }));+ res.write(event("message_stop", {}));+ res.end();+ });+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ return {+ url: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,+ seen,+ close: () => server.close(),+ };+}++function register(baseUrl: string, models: Array<{ id: string; reasoning?: boolean }>, protocol = "anthropic"): void {+ setCustomProviders([+ { id: "direct", name: "Direct", protocol: protocol as "anthropic", baseUrl, models: models.map((m) => ({ ...m })) },+ ]);+}++function replayContext(model: Model<Api>, firstPrompt: string): Context {+ const prior: AssistantMessage = {+ role: "assistant",+ content: [+ { type: "thinking", thinking: "working it out", thinkingSignature: "signed-block" },+ { type: "text", text: "391" },+ ],+ api: model.api,+ provider: model.provider,+ model: model.id,+ usage: {+ input: 1,+ output: 1,+ cacheRead: 0,+ cacheWrite: 0,+ totalTokens: 2,+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },+ },+ stopReason: "stop",+ timestamp: Date.now(),+ };+ return {+ systemPrompt: "terse",+ messages: [+ { role: "user", content: firstPrompt, timestamp: Date.now() },+ prior,+ { role: "user", content: "add 9", timestamp: Date.now() },+ ],+ };+}++async function send(model: Model<Api>, reasoning?: "high"): Promise<void> {+ const runtime = await buildModelRuntime({ direct: "sk-direct" });+ const reply = await runtime+ .streamSimple(model, replayContext(model, "rewritten context: what is 17*23?"), reasoning ? { reasoning } : {})+ .result();+ assert.equal(reply.stopReason, "stop", reply.errorMessage);+}++test("a custom anthropic model of a known adaptive model sends thinking binding even with reasoning off", async () => {+ const upstream = await fakeAnthropic();+ try {+ register(upstream.url, [{ id: "claude-opus-5-5" }]);+ const model = resolveModel("custom/direct/claude-opus-5-5") as Model<Api>;+ await send(model);+ const [call] = upstream.seen;+ assert.ok(call);+ assert.equal(call.body.model, "claude-opus-5-5");+ assert.ok(String(call.headers["anthropic-beta"]).split(",").includes(BINDING_BETA));+ assert.deepEqual(call.body.thinking, { type: "adaptive", display: "summarized", block_binding: DROP_BLOCK });+ const replayed = (call.body.messages as Array<{ content: unknown }>)[1]!.content as Array<{ type: string }>;+ assert.equal(replayed[0]!.type, "thinking");+ } finally {+ upstream.close();+ }+});++test("reasoning on a custom anthropic model of a known adaptive model uses adaptive thinking with binding", async () => {+ const upstream = await fakeAnthropic();+ try {+ register(upstream.url, [{ id: "claude-opus-5-5", reasoning: true }]);+ await send(resolveModel("custom/direct/claude-opus-5-5") as Model<Api>, "high");+ const thinking = upstream.seen[0]!.body.thinking as Record<string, unknown>;+ assert.equal(thinking.type, "adaptive");+ assert.deepEqual(thinking.block_binding, DROP_BLOCK);+ assert.equal(thinking.budget_tokens, undefined);+ } finally {+ upstream.close();+ }+});++test("a custom anthropic model with an unrecognized id keeps the plain request", async () => {+ const upstream = await fakeAnthropic();+ try {+ register(upstream.url, [{ id: "compat-large" }]);+ await send(resolveModel("custom/direct/compat-large") as Model<Api>);+ const [call] = upstream.seen;+ assert.equal(call!.headers["anthropic-beta"]?.includes(BINDING_BETA) ?? false, false);+ assert.equal(call!.body.thinking, undefined);+ } finally {+ upstream.close();+ }+});++test("only anthropic-protocol custom models take on a known model's thinking traits", () => {+ register("https://llm.example.test/v1", [{ id: "claude-opus-5-5" }], "openai");+ const model = resolveModel("custom/direct/claude-opus-5-5");+ assert.equal(model?.api, "openai-completions");+ assert.equal(model?.compat, undefined);+ assert.equal(model?.thinkingLevelMap, undefined);+});++test("a custom model whose provider is gone fails as unavailable instead of sending its id upstream", async () => {+ const upstream = await fakeAnthropic();+ try {+ register(upstream.url, [{ id: "claude-opus-5-5" }]);+ const model = resolveModel("custom/direct/claude-opus-5-5") as Model<Api>;+ const runtime = await buildModelRuntime({ direct: "sk-direct" });+ setCustomProviders([]);+ const reply = await runtime.streamSimple(model, replayContext(model, "hi"), {}).result();+ assert.equal(reply.stopReason, "error");+ assert.match(String(reply.errorMessage), /Model is unavailable: custom\/direct\/claude-opus-5-5/);+ assert.equal((reply as { providerError?: { type?: string } }).providerError?.type, "model_unavailable");+ assert.equal(upstream.seen.length, 0);+ } finally {+ upstream.close();+ }+});