Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Add Claude Haiku 5.5 and move helper/judge models to it (#2109)

Regan Bellbca3b27ffd43
diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts
index 3c0e592be..cab73bd18 100644
--- a/src/harness/claude-harness.ts
+++ b/src/harness/claude-harness.ts
@@ -308,7 +308,7 @@ function effort(level: string | undefined): "low" | "medium" | "high" | "xhigh"
export function createClaudeHarness(opts: ClaudeHarnessOptions = {}): Harness {
const configuredModel = opts.modelId;
- const judgeModelId = opts.judgeModelId ?? "claude-haiku-4-5";
+ const judgeModelId = opts.judgeModelId ?? "claude-haiku-5-5";
const resolveModelId = (scope?: ScopeId) =>
[
typeof configuredModel === "function" ? configuredModel(scope) : configuredModel,
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index d3ecd9677..9bbed274a 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -397,14 +397,16 @@ async function directAnthropicJson(
body: JSON.stringify({
model: gateway?.target ?? requestModel.id,
max_tokens: 64,
+ // Haiku 5.5 thinks by default; a 64-token budget would go entirely to thinking.
+ thinking: { type: "disabled" },
system: systemPrompt,
messages: [{ role: "user", content: userPrompt }],
}),
signal: AbortSignal.timeout(2_000),
});
if (!res.ok) return undefined;
- const json = (await res.json()) as { content?: Array<{ text?: string }> };
- return json.content?.[0]?.text;
+ const json = (await res.json()) as { content?: Array<{ type?: string; text?: string }> };
+ return json.content?.find((block) => block.type === "text")?.text;
}
const APPROVAL_SUMMARY_PROMPT = [
diff --git a/src/model/pi-models.ts b/src/model/pi-models.ts
index daba39807..975f190f0 100644
--- a/src/model/pi-models.ts
+++ b/src/model/pi-models.ts
@@ -190,7 +190,25 @@ export const MODEL_REGISTRY: readonly ModelEntry[] = [
},
},
{ id: "claude-sonnet-5", name: "Claude Sonnet 5", fastMode: false, webui: true, base: true },
- { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", fastMode: false, webui: true, base: true, auxiliary: true },
+ {
+ id: "claude-haiku-5-5",
+ name: "Claude Haiku 5.5",
+ fastMode: false,
+ webui: true,
+ base: true,
+ auxiliary: true,
+ clone: {
+ template: "claude-opus-4-8",
+ input: 0.1,
+ output: 0.5,
+ cacheRead: 0.01,
+ cacheWrite: 0.125,
+ contextWindow: 1_000_000,
+ maxTokens: 128_000,
+ tiers: [{ inputTokensAbove: 100_000, input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 }],
+ },
+ },
+ { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", fastMode: false, webui: true, base: true },
{
id: "gpt-5.6-sol",
buttonLabel: "5.6 Sol",
diff --git a/test/live-slack/harness.ts b/test/live-slack/harness.ts
index 559329dec..3283e13c3 100644
--- a/test/live-slack/harness.ts
+++ b/test/live-slack/harness.ts
@@ -374,13 +374,14 @@ export class Ctx {
body: JSON.stringify({
model: this.env.judgeModel,
max_tokens: 300,
+ thinking: { type: "disabled" },
system: `You are a strict test judge for an AI assistant's Slack replies. Today's date is ${new Date().toISOString().slice(0, 10)}. Answer with exactly PASS or FAIL on the first line, then a one-sentence reason. Judge only what is asked; tone and verbosity are irrelevant unless the question asks about them.`,
messages: [{ role: "user", content: `Question: ${question}\n\nContent to judge:\n${content}` }],
}),
});
- const data = (await res.json()) as { content?: Array<{ text?: string }> };
+ const data = (await res.json()) as { content?: Array<{ type?: string; text?: string }> };
if (!res.ok) throw new Error(`judge call failed: ${res.status} ${JSON.stringify(data).slice(0, 300)}`);
- const verdict = (data.content?.[0]?.text ?? "").trim();
+ const verdict = (data.content?.find((block) => block.type === "text")?.text ?? "").trim();
assert.ok(/^PASS\b/i.test(verdict), `judge failed: ${question}\n${verdict}`);
}
diff --git a/test/live-slack/run.ts b/test/live-slack/run.ts
index 406ec9020..33f3aa062 100644
--- a/test/live-slack/run.ts
+++ b/test/live-slack/run.ts
@@ -56,7 +56,7 @@ async function buildEnv(): Promise<Env> {
qaUserId: qaAuth.userId,
teamId: qaAuth.teamId,
anthropicApiKey: requireEnv("ANTHROPIC_API_KEY"),
- judgeModel: process.env.LIVE_E2E_JUDGE_MODEL ?? "claude-haiku-4-5-20251001",
+ judgeModel: process.env.LIVE_E2E_JUDGE_MODEL ?? "claude-haiku-5-5",
...(process.env.LIVE_E2E_TARGET_CHANNEL ? { targetChannel: process.env.LIVE_E2E_TARGET_CHANNEL } : {}),
sandbox:
process.env.LIVE_E2E_SANDBOX_AVAILABLE === "1" ||
diff --git a/test/model-credential-route.test.ts b/test/model-credential-route.test.ts
index 095501e4f..0139fddca 100644
--- a/test/model-credential-route.test.ts
+++ b/test/model-credential-route.test.ts
@@ -67,6 +67,7 @@ test("admin model credentials are encrypted, write-only, live, and removable", a
{ id: "claude-opus-4-8", name: "Claude Opus 4.8", provider: "anthropic" },
{ id: "claude-sonnet-5-5", name: "Claude Sonnet 5.5", provider: "anthropic" },
{ id: "claude-sonnet-5", name: "Claude Sonnet 5", provider: "anthropic" },
+ { id: "claude-haiku-5-5", name: "Claude Haiku 5.5", provider: "anthropic" },
{ id: "claude-haiku-4-5", name: "Claude Haiku 4.5", provider: "anthropic" },
{ id: "gpt-5.6-sol", name: "GPT-5.6 Sol", provider: "openai" },
{ id: "gpt-5.6-terra", name: "GPT-5.6 Terra", provider: "openai" },
diff --git a/test/model-registry.test.ts b/test/model-registry.test.ts
index ff77ef110..d1247fad4 100644
--- a/test/model-registry.test.ts
+++ b/test/model-registry.test.ts
@@ -135,6 +135,36 @@ test("Sonnet 5.5 never turns thinking off, rejects temperature and fast mode, an
assert.equal(modelSupportedByHarness(id, "codex"), false);
});
+test("Haiku 5.5 offers every effort, keeps thinking off available, rejects temperature and fast mode, and leaves defaults alone", () => {
+ const id = "claude-haiku-5-5";
+ const model = resolveModel(id);
+ assert.ok(model);
+ assert.equal(String(model.provider), "anthropic");
+ assert.equal(model.api, "anthropic-messages");
+ assert.deepEqual(model.thinkingLevelMap, { xhigh: "xhigh", max: "max" });
+ assert.equal((model.compat as { forceAdaptiveThinking?: boolean }).forceAdaptiveThinking, true);
+ assert.equal((model.compat as { supportsTemperature?: boolean }).supportsTemperature, false);
+ assert.equal(safeModelMetadata(id)?.label, "Haiku 5.5");
+ assert.deepEqual(safeModelMetadata(id)?.effortLevelsByHarness.pi, [
+ "auto",
+ "default",
+ "adaptive",
+ "low",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ "ultracode",
+ ]);
+ assert.ok(DEFAULT_WEBUI_MODEL_IDS.includes(id));
+ assert.ok(!FAST_MODE_MODEL_IDS.includes(id));
+ for (const harness of ["pi", "claude", "opencode", "mock"]) {
+ assert.equal(modelSupportedByHarness(id, harness), true);
+ assert.equal(defaultModelForHarness(harness), "claude-opus-5");
+ }
+ assert.equal(modelSupportedByHarness(id, "codex"), false);
+});
+
test("FAST_MODE_MODEL_IDS derives from the registry — the web-ui client reads this, keeps no copy", () => {
assert.deepEqual(
[...FAST_MODE_MODEL_IDS].sort(),
diff --git a/test/native-auto-reasoning.test.ts b/test/native-auto-reasoning.test.ts
index dbf3ab190..9dac6f897 100644
--- a/test/native-auto-reasoning.test.ts
+++ b/test/native-auto-reasoning.test.ts
@@ -111,7 +111,7 @@ test("real Pi AgentSession and Anthropic serializer send Auto without a fixed ef
headers: { "content-type": "text/event-stream" },
});
}) as typeof fetch;
- for (const modelId of [adaptiveId, "claude-opus-5-5", "claude-sonnet-5-5"])
+ for (const modelId of [adaptiveId, "claude-opus-5-5", "claude-sonnet-5-5", "claude-haiku-5-5"])
for (const route of ["api-key", "personal-key", "oauth", "gateway"] as const) {
await t.test(`${modelId}/${route}`, async () => {
const harness = createPiHarness({
diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.ts
index 963531a9e..69a4a640a 100644
--- a/test/pi-harness-fast-mode.test.ts
+++ b/test/pi-harness-fast-mode.test.ts
@@ -107,6 +107,7 @@ const ASTRA = getRequiredModel("gpt-6-astra", false) as Model<Api>;
const SOL_61 = getRequiredModel("gpt-6.1-sol", false) as Model<Api>;
const OPUS_55 = getRequiredModel("claude-opus-5-5", false);
const SONNET_55 = getRequiredModel("claude-sonnet-5-5", false);
+const HAIKU_55 = getRequiredModel("claude-haiku-5-5", false);
const OPUS = getRequiredModel("claude-opus-5", false) as Model<Api>;
const ASTRA_TOKENS = { input: 10_000, output: 2_000, cacheRead: 50_000, cacheWrite: 4_000, totalTokens: 66_000 };
@@ -138,6 +139,19 @@ const pricingCases: Array<[string, Model<Api>, Partial<Usage>, number]> = [
["Opus 5.5 cache reads", OPUS_55, { cacheRead: 100_000 }, 0.02],
["Opus 5.5 1h writes", OPUS_55, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.032],
["Sonnet 5.5 1h writes", SONNET_55, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.016],
+ [
+ "Haiku 5.5 short prompt",
+ HAIKU_55,
+ { input: 10_000, output: 1_000, cacheRead: 40_000, cacheWrite: 16_000, cacheWrite1h: 8_000 },
+ 0.0045,
+ ],
+ ["Haiku 5.5 prompt at the 100k boundary", HAIKU_55, { input: 100_000 }, 0.01],
+ [
+ "Haiku 5.5 long-prompt tier counts cached input",
+ HAIKU_55,
+ { input: 10_000, output: 1_000, cacheRead: 90_000, cacheWrite: 16_000, cacheWrite1h: 8_000 },
+ 0.025,
+ ],
["all 1h writes", OPUS, { cacheWrite: 4_000, cacheWrite1h: 4_000 }, 0.04],
["clamped 1h writes", OPUS, { cacheWrite: 4_000, cacheWrite1h: 40_000 }, 0.04],
];
diff --git a/test/pi-harness-freshness.test.ts b/test/pi-harness-freshness.test.ts
index b6eeed660..b3d6285f1 100644
--- a/test/pi-harness-freshness.test.ts
+++ b/test/pi-harness-freshness.test.ts
@@ -95,17 +95,27 @@ test("ack emoji keeps working on a non-Anthropic base model when an Anthropic ke
resolveProviderKeys: async () => ({ anthropic: "sk-ant-test" }),
});
const realFetch = globalThis.fetch;
- const calls: Array<{ url: string; model: unknown }> = [];
+ const calls: Array<{ url: string; model: unknown; thinking: unknown }> = [];
globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => {
- calls.push({ url: String(url), model: JSON.parse(String(init?.body ?? "{}")).model });
- return new Response(JSON.stringify({ content: [{ type: "text", text: '{"emoji":"eyes"}' }] }), { status: 200 });
+ const body = JSON.parse(String(init?.body ?? "{}"));
+ calls.push({ url: String(url), model: body.model, thinking: body.thinking });
+ return new Response(
+ JSON.stringify({
+ content: [
+ { type: "thinking", thinking: "" },
+ { type: "text", text: '{"emoji":"eyes"}' },
+ ],
+ }),
+ { status: 200 },
+ );
}) as typeof globalThis.fetch;
try {
const picked = await harness.models.pickAckEmoji?.("ship it", ["eyes", "rocket"]);
assert.equal(picked, "eyes", "the pick still lands even though the base model is OpenAI");
assert.equal(calls.length, 1, "the Anthropic ack call was actually attempted");
assert.match(calls[0]!.url, /anthropic/, "it went to the Anthropic API");
- assert.equal(calls[0]!.model, "claude-haiku-4-5", "it used the Anthropic auxiliary, not the OpenAI judge model");
+ assert.equal(calls[0]!.model, "claude-haiku-5-5", "it used the Anthropic auxiliary, not the OpenAI judge model");
+ assert.deepEqual(calls[0]!.thinking, { type: "disabled" }, "the 64-token pick never spends its budget thinking");
} finally {
globalThis.fetch = realFetch;
}
diff --git a/test/pi-models.test.ts b/test/pi-models.test.ts
index 5148e559a..52b89e8c8 100644
--- a/test/pi-models.test.ts
+++ b/test/pi-models.test.ts
@@ -112,6 +112,7 @@ test("the curated catalog contains only current model families", () => {
"claude-opus-4-8",
"claude-sonnet-5-5",
"claude-sonnet-5",
+ "claude-haiku-5-5",
"claude-haiku-4-5",
"gpt-5.6-sol",
"gpt-5.6-terra",
@@ -164,13 +165,13 @@ test("the curated catalog contains only current model families", () => {
test("auxiliary models come from the configured base model's own provider", () => {
assert.equal(
auxiliaryModelFor("claude-opus-5"),
- "claude-haiku-4-5",
+ "claude-haiku-5-5",
"the deployment default resolves an Anthropic auxiliary",
);
- assert.equal(auxiliaryModelFor("claude-opus-5-5"), "claude-haiku-4-5");
- assert.equal(auxiliaryModelFor("claude-opus-4-8"), "claude-haiku-4-5");
- assert.equal(auxiliaryModelFor("claude-fable-5-1"), "claude-haiku-4-5");
- assert.equal(auxiliaryModelFor("claude-fable-5"), "claude-haiku-4-5");
+ assert.equal(auxiliaryModelFor("claude-opus-5-5"), "claude-haiku-5-5");
+ assert.equal(auxiliaryModelFor("claude-opus-4-8"), "claude-haiku-5-5");
+ assert.equal(auxiliaryModelFor("claude-fable-5-1"), "claude-haiku-5-5");
+ assert.equal(auxiliaryModelFor("claude-fable-5"), "claude-haiku-5-5");
assert.equal(
auxiliaryModelFor("gpt-5.6-sol"),
"gpt-5.6-luna",
@@ -180,7 +181,7 @@ test("auxiliary models come from the configured base model's own provider", () =
});
test("the Anthropic auxiliary is resolvable by provider, so Anthropic-only surfaces keep working", () => {
- assert.equal(auxiliaryModelForProvider("anthropic"), "claude-haiku-4-5");
+ assert.equal(auxiliaryModelForProvider("anthropic"), "claude-haiku-5-5");
assert.equal(auxiliaryModelForProvider("openai"), "gpt-5.6-luna");
assert.equal(auxiliaryModelForProvider("nope"), undefined);
});
@@ -244,6 +245,17 @@ test("context token budget is half of each model's real input room", () => {
assert.equal(sonnet55.contextWindow, 1_000_000);
assert.equal(sonnet55.maxTokens, 128_000);
assert.deepEqual(sonnet55.cost, { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5, tiers: undefined });
+ const haiku55 = getRequiredModel("claude-haiku-5-5");
+ assert.equal(haiku55.contextWindow, 1_000_000);
+ assert.equal(haiku55.maxTokens, 128_000);
+ assert.deepEqual(haiku55.cost, {
+ input: 0.1,
+ output: 0.5,
+ cacheRead: 0.01,
+ cacheWrite: 0.125,
+ tiers: [{ inputTokensAbove: 100_000, input: 0.5, output: 2.5, cacheRead: 0.05, cacheWrite: 0.625 }],
+ });
+ assert.equal(contextTokenBudgetForModel("claude-haiku-5-5"), 150_000);
assert.equal(getRequiredModel("claude-fable-5").contextWindow, 1_000_000);
assert.equal(contextTokenBudgetForModel("claude-fable-5"), 150_000);
assert.equal(contextTokenBudgetForModel("gpt-5.6-sol"), 150_000);
diff --git a/test/pi-opus-55.test.ts b/test/pi-opus-55.test.ts
index b627c3386..8e7037f4f 100644
--- a/test/pi-opus-55.test.ts
+++ b/test/pi-opus-55.test.ts
@@ -10,7 +10,7 @@ for (const modelId of ["claude-opus-5-5", "claude-sonnet-5-5"])
test(`${modelId} wire requests keep mandatory thinking, caller hooks and beta headers across Pi entry points`, (t) =>
assertMandatoryThinkingWire(t, modelId));
-async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Promise<void> {
+test("claude-haiku-5-5 wire requests bind adaptive thinking at every effort and turn thinking off only without one", async (t) => {
const originalFetch = globalThis.fetch;
t.after(() => {
globalThis.fetch = originalFetch;
@@ -18,27 +18,65 @@ async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Pro
const requests: Array<{ headers: Headers; body: Record<string, unknown> }> = [];
globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
requests.push({ headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) });
- const events = [
- {
- type: "message_start",
- message: {
- id: "msg_test",
- type: "message",
- role: "assistant",
- model: modelId,
- content: [],
- usage: { input_tokens: 1, output_tokens: 0 },
- },
- },
- { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
- { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } },
- { type: "content_block_stop", index: 0 },
- { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } },
- { type: "message_stop" },
- ];
- return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), {
- headers: { "content-type": "text/event-stream" },
+ return okStream("claude-haiku-5-5");
+ }) as typeof fetch;
+ const model = getRequiredModel("claude-haiku-5-5", false);
+ const runtime = await buildModelRuntime({ anthropic: "sk-offline-test-key" });
+ for (const effort of ["low", "medium", "high", "xhigh", "max"] as const) {
+ const response = await runtime.streamSimple(model, context, { reasoning: effort, maxRetries: 0 }).result();
+ assert.equal(response.stopReason, "stop", response.errorMessage);
+ const request = requests.at(-1)!;
+ assert.ok(request.headers.get("anthropic-beta")?.includes(bindingBeta));
+ assert.deepEqual(request.body.thinking, {
+ type: "adaptive",
+ display: "summarized",
+ block_binding: { prefix_mismatch_behavior: "drop_block" },
});
+ assert.deepEqual(request.body.output_config, { effort });
+ assert.equal(request.body.temperature, undefined);
+ }
+ // Without an effort the API default (medium) applies, where Haiku 5.5 accepts thinking off.
+ await probeModel(model, { anthropic: "sk-offline-test-key" }, AbortSignal.timeout(10_000), false);
+ const probe = requests.at(-1)!;
+ assert.deepEqual(probe.body.thinking, { type: "disabled" });
+ assert.equal(probe.body.output_config, undefined);
+ assert.equal(probe.body.speed, undefined);
+ assert.ok(!probe.headers.get("anthropic-beta")?.includes("fast-mode-2026-02-01"));
+});
+
+function okStream(modelId: string): Response {
+ const events = [
+ {
+ type: "message_start",
+ message: {
+ id: "msg_test",
+ type: "message",
+ role: "assistant",
+ model: modelId,
+ content: [],
+ usage: { input_tokens: 1, output_tokens: 0 },
+ },
+ },
+ { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } },
+ { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "OK" } },
+ { type: "content_block_stop", index: 0 },
+ { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } },
+ { type: "message_stop" },
+ ];
+ return new Response(events.map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join(""), {
+ headers: { "content-type": "text/event-stream" },
+ });
+}
+
+async function assertMandatoryThinkingWire(t: TestContext, modelId: string): Promise<void> {
+ const originalFetch = globalThis.fetch;
+ t.after(() => {
+ globalThis.fetch = originalFetch;
+ });
+ const requests: Array<{ headers: Headers; body: Record<string, unknown> }> = [];
+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+ requests.push({ headers: new Headers(init?.headers), body: JSON.parse(String(init?.body)) });
+ return okStream(modelId);
}) as typeof fetch;
const model = { ...getRequiredModel(modelId, false), headers: { "anthropic-beta": "prior-beta" } };
const runtime = await buildModelRuntime({ anthropic: "sk-offline-test-key" });
diff --git a/test/surface-cache-ambient.test.ts b/test/surface-cache-ambient.test.ts
index 68363ab0a..36bc867f2 100644
--- a/test/surface-cache-ambient.test.ts
+++ b/test/surface-cache-ambient.test.ts
@@ -515,7 +515,7 @@ test("a judged batch records a durable judgment row: prompt, decision, model, ts
assert.equal(rows.length, 1, "one row for the judged batch");
assert.equal(rows[0]!.decision, "act");
assert.ok(rows[0]!.prompt && rows[0]!.prompt.includes("NEW MESSAGES"));
- assert.equal(rows[0]!.model, "claude-haiku-4-5", "the resolved judge model is recorded (default Haiku)");
+ assert.equal(rows[0]!.model, "claude-haiku-5-5", "the resolved judge model is recorded (default Haiku)");
assert.equal(rows[0]!.tsFrom, "5.0");
assert.equal(rows[0]!.tsTo, "5.0");
assert.ok(typeof rows[0]!.latencyMs === "number");