Multiplayer agent harness for work.
Add Claude Haiku 5.5 and move helper/judge models to it (#2109)
Regan Bellbca3b27ffd43
diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.tsindex 3c0e592be..cab73bd18 100644--- a/src/harness/claude-harness.ts+++ b/src/harness/claude-harness.ts@@ -308,7 +308,7 @@ function effort(level: string | undefined): "low" | "medium" | "high" | "xhigh"export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {const configuredModel = opts.modelId;- const judgeModelId = opts.judgeModelId ?? "claude-haiku-4-5";+ const judgeModelId = opts.judgeModelId ?? "claude-haiku-5-5";const resolveModelId = (scope?: ScopeId) =>[typeof configuredModel === "function" ? configuredModel(scope) : configuredModel,diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex d3ecd9677..9bbed274a 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -397,14 +397,16 @@ async function directAnthropicJson(body: JSON.stringify({model: gateway?.target ?? requestModel.id,max_tokens: 64,+ // Haiku 5.5 thinks by default; a 64-token budget would go entirely to thinking.+ thinking: { type: "disabled" },system: systemPrompt,messages: [{ role: "user", content: userPrompt }],}),signal: AbortSignal.timeout(2_000),});if (!res.ok) return undefined;- const json = (await res.json()) as { content?: Array<{ text?: string }> };- return json.content?.[0]?.text;+ const json = (await res.json()) as { content?: Array<{ type?: string; text?: string }> };+ return json.content?.find((block) => block.type === "text")?.text;}const APPROVAL_SUMMARY_PROMPT = [diff --git a/src/model/pi-models.ts b/src/model/pi-models.tsindex daba39807..975f190f0 100644--- a/src/model/pi-models.ts+++ b/src/model/pi-models.ts@@ -190,7 +190,25 @@ export const MODEL_REGISTRY: readonly ModelEntry[] = [},},{ id: "claude-sonnet-5", name: "Claude Sonnet 5", fastMode: false, webui: true, base: true },- { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", fastMode: false, webui: true, base: true, auxiliary: true },+ {+ id: "claude-haiku-5-5",+ name: "Claude Haiku 5.5",+ fastMode: false,+ webui: true,+ base: true,+ auxiliary: true,+ clone: {+ template: "claude-opus-4-8",+ input: 0.1,+ output: 0.5,+ cacheRead: 0.01,+ cacheWrite: 0.125,+ contextWindow: 1_000_000,+ maxTokens: 128_000,+ tiers: [{ inputTokensAbove: 100_000, input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 }],+ },+ },+ { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", fastMode: false, webui: true, base: true },{id: "gpt-5.6-sol",buttonLabel: "5.6 Sol",diff --git a/test/live-slack/harness.ts b/test/live-slack/harness.tsindex 559329dec..3283e13c3 100644--- a/test/live-slack/harness.ts+++ b/test/live-slack/harness.ts@@ -374,13 +374,14 @@ export class Ctx {body: JSON.stringify({model: this.env.judgeModel,max_tokens: 300,+ thinking: { type: "disabled" },system: `You are a strict test judge for an AI assistant's Slack replies. Today's date is ${new Date().toISOString().slice(0, 10)}. Answer with exactly PASS or FAIL on the first line, then a one-sentence reason. Judge only what is asked; tone and verbosity are irrelevant unless the question asks about them.`,messages: [{ role: "user", content: `Question: ${question}\n\nContent to judge:\n${content}` }],}),});- const data = (await res.json()) as { content?: Array<{ text?: string }> };+ const data = (await res.json()) as { content?: Array<{ type?: string; text?: string }> };if (!res.ok) throw new Error(`judge call failed: ${res.status} ${JSON.stringify(data).slice(0, 300)}`);- const verdict = (data.content?.[0]?.text ?? "").trim();+ const verdict = (data.content?.find((block) => block.type === "text")?.text ?? "").trim();assert.ok(/^PASS\b/i.test(verdict), `judge failed: ${question}\n${verdict}`);}diff --git a/test/live-slack/run.ts b/test/live-slack/run.tsindex 406ec9020..33f3aa062 100644--- a/test/live-slack/run.ts+++ b/test/live-slack/run.ts@@ -56,7 +56,7 @@ async function buildEnv(): Promise<Env> {qaUserId: qaAuth.userId,teamId: qaAuth.teamId,anthropicApiKey: requireEnv("ANTHROPIC_API_KEY"),- judgeModel: process.env.LIVE_E2E_JUDGE_MODEL ?? "claude-haiku-4-5-20251001",+ judgeModel: process.env.LIVE_E2E_JUDGE_MODEL ?? "claude-haiku-5-5",...(process.env.LIVE_E2E_TARGET_CHANNEL ? { targetChannel: process.env.LIVE_E2E_TARGET_CHANNEL } : {}),sandbox:process.env.LIVE_E2E_SANDBOX_AVAILABLE === "1" ||diff --git a/test/model-credential-route.test.ts b/test/model-credential-route.test.tsindex 095501e4f..0139fddca 100644--- a/test/model-credential-route.test.ts+++ b/test/model-credential-route.test.ts@@ -67,6 +67,7 @@ test("admin model credentials are encrypted, write-only, live, and removable", a{ id: "claude-opus-4-8", name: "Claude Opus 4.8", provider: "anthropic" },{ id: "claude-sonnet-5-5", name: "Claude Sonnet 5.5", provider: "anthropic" },{ id: "claude-sonnet-5", name: "Claude Sonnet 5", provider: "anthropic" },+ { id: "claude-haiku-5-5", name: "Claude Haiku 5.5", provider: "anthropic" },{ id: "claude-haiku-4-5", name: "Claude Haiku 4.5", provider: "anthropic" },{ id: "gpt-5.6-sol", name: "GPT-5.6 Sol", provider: "openai" },{ id: "gpt-5.6-terra", name: "GPT-5.6 Terra", provider: "openai" },diff --git a/test/model-registry.test.ts b/test/model-registry.test.tsindex ff77ef110..d1247fad4 100644--- a/test/model-registry.test.ts+++ b/test/model-registry.test.ts@@ -135,6 +135,36 @@ test("Sonnet 5.5 never turns thinking off, rejects temperature and fast mode, anassert.equal(modelSupportedByHarness(id, "codex"), false);});+test("Haiku 5.5 offers every effort, keeps thinking off available, rejects temperature and fast mode, and leaves defaults alone", () => {+ const id = "claude-haiku-5-5";+ const model = resolveModel(id);+ assert.ok(model);+ assert.equal(String(model.provider), "anthropic");+ assert.equal(model.api, "anthropic-messages");+ assert.deepEqual(model.thinkingLevelMap, { xhigh: "xhigh", max: "max" });+ assert.equal((model.compat as { forceAdaptiveThinking?: boolean }).forceAdaptiveThinking, true);+ assert.equal((model.compat as { supportsTemperature?: boolean }).supportsTemperature, false);+ assert.equal(safeModelMetadata(id)?.label, "Haiku 5.5");+ assert.deepEqual(safeModelMetadata(id)?.effortLevelsByHarness.pi, [+ "auto",+ "default",+ "adaptive",+ "low",+ "medium",+ "high",+ "xhigh",+ "max",+ "ultracode",+ ]);+ assert.ok(DEFAULT_WEBUI_MODEL_IDS.includes(id));+ assert.ok(!FAST_MODE_MODEL_IDS.includes(id));+ for (const harness of ["pi", "claude", "opencode", "mock"]) {+ assert.equal(modelSupportedByHarness(id, harness), true);+ assert.equal(defaultModelForHarness(harness), "claude-opus-5");+ }+ assert.equal(modelSupportedByHarness(id, "codex"), false);+});+test("FAST_MODE_MODEL_IDS derives from the registry — the web-ui client reads this, keeps no copy", () => {assert.deepEqual([...FAST_MODE_MODEL_IDS].sort(),diff --git a/test/native-auto-reasoning.test.ts b/test/native-auto-reasoning.test.tsindex dbf3ab190..9dac6f897 100644--- a/test/native-auto-reasoning.test.ts+++ b/test/native-auto-reasoning.test.ts@@ -111,7 +111,7 @@ test("real Pi AgentSession and Anthropic serializer send Auto without a fixed efheaders: { "content-type": "text/event-stream" },});}) as typeof fetch;- for (const modelId of [adaptiveId, "claude-opus-5-5", "claude-sonnet-5-5"])+ for (const modelId of [adaptiveId, "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"])for (const route of ["api-key", "personal-key", "oauth", "gateway"] as const) {await t.test(`${modelId}/${route}`, async () => {const harness = createPiHarness({diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.tsindex 963531a9e..69a4a640a 100644--- a/test/pi-harness-fast-mode.test.ts+++ b/test/pi-harness-fast-mode.test.ts@@ -107,6 +107,7 @@ const ASTRA = getRequiredModel("gpt-6-astra", false) as Model<Api>;const SOL_61 = getRequiredModel("gpt-6.1-sol", false) as Model<Api>;const OPUS_55 = getRequiredModel("claude-opus-5-5", false);const SONNET_55 = getRequiredModel("claude-sonnet-5-5", false);+const HAIKU_55 = getRequiredModel("claude-haiku-5-5", false);const OPUS = getRequiredModel("claude-opus-5", false) as Model<Api>;const ASTRA_TOKENS = { input: 10_000, output: 2_000, cacheRead: 50_000, cacheWrite: 4_000, totalTokens: 66_000 };@@ -138,6 +139,19 @@ const pricingCases: Array<[string, Model<Api>, Partial<Usage>, number]> = [["Opus 5.5 cache reads", OPUS_55, { cacheRead: 100_000 }, 0.02],["Opus 5.5 1h writes", OPUS_55, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.032],["Sonnet 5.5 1h writes", SONNET_55, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.016],+ [+ "Haiku 5.5 short prompt",+ HAIKU_55,+ { input: 10_000, output: 1_000, cacheRead: 40_000, cacheWrite: 16_000, cacheWrite1h: 8_000 },+ 0.0045,+ ],+ ["Haiku 5.5 prompt at the 100k boundary", HAIKU_55, { input: 100_000 }, 0.01],+ [+ "Haiku 5.5 long-prompt tier counts cached input",+ HAIKU_55,+ { input: 10_000, output: 1_000, cacheRead: 90_000, cacheWrite: 16_000, cacheWrite1h: 8_000 },+ 0.025,+ ],["all 1h writes", OPUS, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.04],["clamped 1h writes", OPUS, { cacheWrite: 4_000, cacheWrite1h: 40_000 }, 0.04],];diff --git a/test/pi-harness-freshness.test.ts b/test/pi-harness-freshness.test.tsindex b6eeed660..b3d6285f1 100644--- a/test/pi-harness-freshness.test.ts+++ b/test/pi-harness-freshness.test.ts@@ -95,17 +95,27 @@ test("ack emoji keeps working on a non-Anthropic base model when an Anthropic keresolveProviderKeys: async () => ({ anthropic: "sk-ant-test" }),});const realFetch = globalThis.fetch;- const calls: Array<{ url: string; model: unknown }> = [];+ const calls: Array<{ url: string; model: unknown; thinking: unknown }> = [];globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => {- calls.push({ url: String(url), model: JSON.parse(String(init?.body ?? "{}")).model });- return new Response(JSON.stringify({ content: [{ type: "text", text: '{"emoji":"eyes"}' }] }), { status: 200 });+ const body = JSON.parse(String(init?.body ?? "{}"));+ calls.push({ url: String(url), model: body.model, thinking: body.thinking });+ return new Response(+ JSON.stringify({+ content: [+ { type: "thinking", thinking: "" },+ { type: "text", text: '{"emoji":"eyes"}' },+ ],+ }),+ { status: 200 },+ );}) as typeof globalThis.fetch;try {const picked = await harness.models.pickAckEmoji?.("ship it", ["eyes", "rocket"]);assert.equal(picked, "eyes", "the pick still lands even though the base model is OpenAI");assert.equal(calls.length, 1, "the Anthropic ack call was actually attempted");assert.match(calls[0]!.url, /anthropic/, "it went to the Anthropic API");- assert.equal(calls[0]!.model, "claude-haiku-4-5", "it used the Anthropic auxiliary, not the OpenAI judge model");+ assert.equal(calls[0]!.model, "claude-haiku-5-5", "it used the Anthropic auxiliary, not the OpenAI judge model");+ assert.deepEqual(calls[0]!.thinking, { type: "disabled" }, "the 64-token pick never spends its budget thinking");} finally {globalThis.fetch = realFetch;}diff --git a/test/pi-models.test.ts b/test/pi-models.test.tsindex 5148e559a..52b89e8c8 100644--- a/test/pi-models.test.ts+++ b/test/pi-models.test.ts@@ -112,6 +112,7 @@ test("the curated catalog contains only current model families", () => {"claude-opus-4-8","claude-sonnet-5-5","claude-sonnet-5",+ "claude-haiku-5-5","claude-haiku-4-5","gpt-5.6-sol","gpt-5.6-terra",@@ -164,13 +165,13 @@ test("the curated catalog contains only current model families", () => {test("auxiliary models come from the configured base model's own provider", () => {assert.equal(auxiliaryModelFor("claude-opus-5"),- "claude-haiku-4-5",+ "claude-haiku-5-5","the deployment default resolves an Anthropic auxiliary",);- assert.equal(auxiliaryModelFor("claude-opus-5-5"), "claude-haiku-4-5");- assert.equal(auxiliaryModelFor("claude-opus-4-8"), "claude-haiku-4-5");- assert.equal(auxiliaryModelFor("claude-fable-5-1"), "claude-haiku-4-5");- assert.equal(auxiliaryModelFor("claude-fable-5"), "claude-haiku-4-5");+ assert.equal(auxiliaryModelFor("claude-opus-5-5"), "claude-haiku-5-5");+ assert.equal(auxiliaryModelFor("claude-opus-4-8"), "claude-haiku-5-5");+ assert.equal(auxiliaryModelFor("claude-fable-5-1"), "claude-haiku-5-5");+ assert.equal(auxiliaryModelFor("claude-fable-5"), "claude-haiku-5-5");assert.equal(auxiliaryModelFor("gpt-5.6-sol"),"gpt-5.6-luna",@@ -180,7 +181,7 @@ test("auxiliary models come from the configured base model's own provider", () =});test("the Anthropic auxiliary is resolvable by provider, so Anthropic-only surfaces keep working", () => {- assert.equal(auxiliaryModelForProvider("anthropic"), "claude-haiku-4-5");+ assert.equal(auxiliaryModelForProvider("anthropic"), "claude-haiku-5-5");assert.equal(auxiliaryModelForProvider("openai"), "gpt-5.6-luna");assert.equal(auxiliaryModelForProvider("nope"), undefined);});@@ -244,6 +245,17 @@ test("context token budget is half of each model's real input room", () => {assert.equal(sonnet55.contextWindow, 1_000_000);assert.equal(sonnet55.maxTokens, 128_000);assert.deepEqual(sonnet55.cost, { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5, tiers: undefined });+ const haiku55 = getRequiredModel("claude-haiku-5-5");+ assert.equal(haiku55.contextWindow, 1_000_000);+ assert.equal(haiku55.maxTokens, 128_000);+ assert.deepEqual(haiku55.cost, {+ input: 0.1,+ output: 0.5,+ cacheRead: 0.01,+ cacheWrite: 0.125,+ tiers: [{ inputTokensAbove: 100_000, input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 }],+ });+ assert.equal(contextTokenBudgetForModel("claude-haiku-5-5"), 150_000);assert.equal(getRequiredModel("claude-fable-5").contextWindow, 1_000_000);assert.equal(contextTokenBudgetForModel("claude-fable-5"), 150_000);assert.equal(contextTokenBudgetForModel("gpt-5.6-sol"), 150_000);diff --git a/test/pi-opus-55.test.ts b/test/pi-opus-55.test.tsindex b627c3386..8e7037f4f 100644--- a/test/pi-opus-55.test.ts+++ b/test/pi-opus-55.test.ts@@ -10,7 +10,7 @@ for (const modelId of ["claude-opus-5-5", "claude-sonnet-5-5"])test(`${modelId} wire requests keep mandatory thinking, caller hooks and beta headers across Pi entry points`, (t) =>assertMandatoryThinkingWire(t, modelId));-async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Promise<void> {+test("claude-haiku-5-5 wire requests bind adaptive thinking at every effort and turn thinking off only without one", async (t) => {const originalFetch = globalThis.fetch;t.after(() => {globalThis.fetch = originalFetch;@@ -18,27 +18,65 @@ async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Proconst requests: Array<{ headers: Headers; body: Record<string, unknown> }> = [];globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {requests.push({ headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) });- const events = [- {- type: "message_start",- message: {- id: "msg_test",- type: "message",- role: "assistant",- model: modelId,- content: [],- usage: { input_tokens: 1, output_tokens: 0 },- },- },- { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },- { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } },- { type: "content_block_stop", index: 0 },- { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } },- { type: "message_stop" },- ];- return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), {- headers: { "content-type": "text/event-stream" },+ return okStream("claude-haiku-5-5");+ }) as typeof fetch;+ const model = getRequiredModel("claude-haiku-5-5", false);+ const runtime = await buildModelRuntime({ anthropic: "sk-offline-test-key" });+ for (const effort of ["low", "medium", "high", "xhigh", "max"] as const) {+ const response = await runtime.streamSimple(model, context, { reasoning: effort, maxRetries: 0 }).result();+ assert.equal(response.stopReason, "stop", response.errorMessage);+ const request = requests.at(-1)!;+ assert.ok(request.headers.get("anthropic-beta")?.includes(bindingBeta));+ assert.deepEqual(request.body.thinking, {+ type: "adaptive",+ display: "summarized",+ block_binding: { prefix_mismatch_behavior: "drop_block" },});+ assert.deepEqual(request.body.output_config, { effort });+ assert.equal(request.body.temperature, undefined);+ }+ // Without an effort the API default (medium) applies, where Haiku 5.5 accepts thinking off.+ await probeModel(model, { anthropic: "sk-offline-test-key" }, AbortSignal.timeout(10_000), false);+ const probe = requests.at(-1)!;+ assert.deepEqual(probe.body.thinking, { type: "disabled" });+ assert.equal(probe.body.output_config, undefined);+ assert.equal(probe.body.speed, undefined);+ assert.ok(!probe.headers.get("anthropic-beta")?.includes("fast-mode-2026-02-01"));+});++function okStream(modelId: string): Response {+ const events = [+ {+ type: "message_start",+ message: {+ id: "msg_test",+ type: "message",+ role: "assistant",+ model: modelId,+ content: [],+ usage: { input_tokens: 1, output_tokens: 0 },+ },+ },+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } },+ { type: "content_block_stop", index: 0 },+ { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } },+ { type: "message_stop" },+ ];+ return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), {+ headers: { "content-type": "text/event-stream" },+ });+}++async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Promise<void> {+ const originalFetch = globalThis.fetch;+ t.after(() => {+ globalThis.fetch = originalFetch;+ });+ const requests: Array<{ headers: Headers; body: Record<string, unknown> }> = [];+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {+ requests.push({ headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) });+ return okStream(modelId);}) as typeof fetch;const model = { ...getRequiredModel(modelId, false), headers: { "anthropic-beta": "prior-beta" } };const runtime = await buildModelRuntime({ anthropic: "sk-offline-test-key" });diff --git a/test/surface-cache-ambient.test.ts b/test/surface-cache-ambient.test.tsindex 68363ab0a..36bc867f2 100644--- a/test/surface-cache-ambient.test.ts+++ b/test/surface-cache-ambient.test.ts@@ -515,7 +515,7 @@ test("a judged batch records a durable judgment row: prompt, decision, model, tsassert.equal(rows.length, 1, "one row for the judged batch");assert.equal(rows[0]!.decision, "act");assert.ok(rows[0]!.prompt && rows[0]!.prompt.includes("NEW MESSAGES"));- assert.equal(rows[0]!.model, "claude-haiku-4-5", "the resolved judge model is recorded (default Haiku)");+ assert.equal(rows[0]!.model, "claude-haiku-5-5", "the resolved judge model is recorded (default Haiku)");assert.equal(rows[0]!.tsFrom, "5.0");assert.equal(rows[0]!.tsTo, "5.0");assert.ok(typeof rows[0]!.latencyMs === "number");