Multiplayer agent harness for work.
fix: preserve model capabilities when mapping request options (#1395)
Joshua France1519cdc0553f
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex fb5373adb..68e9e1e96 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -17,6 +17,7 @@ import {type Api,type Context,type Model,+ type ModelThinkingLevel,type ModelsApiStreamOptions,type ModelsSimpleStreamOptions,type ProviderHeaders,@@ -24,7 +25,6 @@ import {import { baseModelProviders, CONFIG_DEFAULTS, type Config } from "../config.ts";type LegacyThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh";-const LEGACY_THINKING_LEVELS = new Set<string>(["off", "minimal", "low", "medium", "high", "xhigh"]);const TURN_EFFORT_LEVELS = new Set<string>(["off","minimal",@@ -1356,12 +1356,6 @@ export function wantsFastMode(fastMode: boolean | undefined, modelId: string | ureturn fastMode === true && modelSupportsFastMode(modelId);}-export const TURN_PROVIDER_EFFORT_ALIASES: Record<string, string | null> = {- max: "max",- ultracode: "max",- auto: null,-};-export function applyFastSpeed<T>(payload: T, fast: boolean | undefined, api?: string): T {if (fast && payload && typeof payload === "object") {if (api && api.toLowerCase().startsWith("openai")) {@@ -1406,9 +1400,10 @@ function estimatePayloadTokens(payload: Record<string, unknown>): number | undefexport function guardOutputBudget(payload: unknown, model: unknown): OutputBudgetGuardResult {const p = payload as Record<string, unknown> | null;if (!p || typeof p !== "object") return { kind: "ok" };- let capKey: "max_tokens" | "max_output_tokens" | undefined;+ let capKey: "max_tokens" | "max_output_tokens" | "max_completion_tokens" | undefined;if (typeof p.max_tokens === "number") capKey = "max_tokens";else if (typeof p.max_output_tokens === "number") capKey = "max_output_tokens";+ else if (typeof p.max_completion_tokens === "number") capKey = "max_completion_tokens";if (capKey === undefined) return { kind: "ok" };const cap = p[capKey] as number;if (cap >= OUTPUT_BUDGET_FLOOR_TOKENS) return { kind: "ok" };@@ -1454,30 +1449,15 @@ function withFastModeHeaders(model: Model<Api>): Model<Api> {};}-function applyEffortAliases(model: unknown): void {- const mutable = model as { thinkingLevelMap?: Record<string, string | null> } | undefined;- if (!mutable) return;- mutable.thinkingLevelMap = {- ...mutable.thinkingLevelMap,- ...TURN_PROVIDER_EFFORT_ALIASES,- };-}-export function applyTurnEffort(session: AgentSession, level?: string): void {if (!level || !TURN_EFFORT_LEVELS.has(level)) return;const effectiveLevel =level === "auto" && session.state.model ? defaultInteractiveThinkingLevel(session.state.model) : level;const normalizedLevel = effectiveLevel === "auto" ? "medium" : effectiveLevel;- applyEffortAliases(session.state.model);- try {- if (LEGACY_THINKING_LEVELS.has(normalizedLevel)) {- session.setThinkingLevel(normalizedLevel as LegacyThinkingLevel);- } else {- session.state.thinkingLevel = normalizedLevel as typeof session.state.thinkingLevel;- }- } catch (e) {- swallow("pi: set thinking level", e);- }+ const providerLevel = normalizedLevel === "ultracode" ? "max" : normalizedLevel;+ // Normalize UI aliases before Pi clamps to the model's declared capabilities.+ // Mutating thinkingLevelMap would enable efforts the provider explicitly excludes.+ session.setThinkingLevel(providerLevel as ModelThinkingLevel);}export function createPiHarness(opts?: PiHarnessOptions): Harness {diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.tsindex 843211dd9..758a8ce27 100644--- a/src/model/custom-providers.ts+++ b/src/model/custom-providers.ts@@ -69,6 +69,12 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {for (const [field, v] of [["contextWindow", m.contextWindow],["maxTokens", m.maxTokens],+ ] as const) {+ if (v !== undefined && (!Number.isSafeInteger(v) || v <= 0)) {+ throw new Error(`model "${m.id}": ${field} must be a positive safe integer`);+ }+ }+ for (const [field, v] of [["input", m.input],["output", m.output],] as const) {diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.tsindex dfe3726fa..afcbe264b 100644--- a/test/custom-providers.test.ts+++ b/test/custom-providers.test.ts@@ -206,3 +206,21 @@ test("catalog cache invalidates immediately when the custom registry changes", aconst cleared = await selectableModelCatalog(fetcher);assert.ok(!cleared.some((m) => m.id === "fresh-model"), "removal visible immediately too");});++for (const field of ["contextWindow", "maxTokens"] as const) {+ test(`custom provider ${field} must be a positive safe integer`, () => {+ for (const value of [0, -1, 2048.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity]) {+ assert.throws(+ () => validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", [field]: value }] }),+ /positive safe integer/,+ `${field}=${value}`,+ );+ }+ assert.doesNotThrow(() =>+ validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", [field]: 8192 }] }),+ );+ assert.doesNotThrow(() =>+ validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", input: 0.25, output: 0 }] }),+ );+ });+}diff --git a/test/pi-harness-effort-wire.test.ts b/test/pi-harness-effort-wire.test.tsnew file mode 100644index 000000000..2d8fb1625--- /dev/null+++ b/test/pi-harness-effort-wire.test.ts@@ -0,0 +1,110 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { createServer } from "node:http";+import { createPiHarness } from "../src/harness/pi-harness.ts";+import { getRequiredModel } from "../src/model/pi-models.ts";+import type { HarnessTurnInput } from "../src/harness/harness.ts";++// Exercise the real AgentSession and SDK serializer, not just the effort setter.+for (const [modelId, upper, allowed] of [+ ["gpt-5.4", "xhigh", ["none", "low", "medium", "high", "xhigh"]],+ ["gpt-5", "high", ["minimal", "low", "medium", "high"]],+] as const) {+ test(`Pi sends supported reasoning through HTTP for ${modelId}`, async (t) => {+ const bodies: Array<{ reasoning?: { effort?: string } }> = [];+ const server = createServer(async (req, res) => {+ let raw = "";+ for await (const chunk of req) raw += String(chunk);+ const body = JSON.parse(raw);+ bodies.push(body);+ if (!(allowed as readonly string[]).includes(body.reasoning?.effort)) {+ res.writeHead(400, { "content-type": "application/json" });+ res.end(JSON.stringify({ error: { message: "Unsupported reasoning.effort", type: "invalid_request_error" } }));+ return;+ }+ res.writeHead(200, { "content-type": "text/event-stream" });+ const message = {+ id: "msg_test",+ type: "message",+ role: "assistant",+ status: "completed",+ content: [{ type: "output_text", text: "ok", annotations: [] }],+ };+ for (const event of [+ { type: "response.created", response: { id: "resp_test", model: modelId, status: "in_progress", output: [] } },+ { type: "response.output_item.added", output_index: 0, item: { ...message, content: [] } },+ {+ type: "response.content_part.added",+ output_index: 0,+ content_index: 0,+ part: { type: "output_text", text: "", annotations: [] },+ },+ { type: "response.output_text.delta", output_index: 0, content_index: 0, delta: "ok" },+ { type: "response.output_item.done", output_index: 0, item: message },+ {+ type: "response.completed",+ response: {+ id: "resp_test",+ model: modelId,+ status: "completed",+ output: [message],+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },+ },+ },+ ])+ res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);+ res.end();+ });+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ t.after(() => new Promise<void>((resolve, reject) => server.close((e) => (e ? reject(e) : resolve()))));+ const address = server.address();+ assert(address && typeof address !== "string");+ const originalMap = structuredClone(getRequiredModel(modelId).thinkingLevelMap);+ const harness = createPiHarness({+ defaultModelId: modelId,+ modelGateway: {+ url: `http://127.0.0.1:${address.port}`,+ apiKey: "test-key",+ apiKeyHeader: "api-key",+ models: { [modelId]: modelId },+ },+ });+ t.after(() => harness.turns.close?.());+ // Repeat turns in the same conversation with different effort selections.+ for (const [effortLevel, expected] of [+ ["ultracode", upper],+ ["max", upper],+ ["medium", "medium"],+ ["low", "low"],+ ["high", "high"],+ ["minimal", modelId === "gpt-5" ? "minimal" : "low"],+ ["off", modelId === "gpt-5" ? "minimal" : "none"],+ ["xhigh", upper],+ ["auto", "medium"],+ ]) {+ await t.test(effortLevel!, async () => {+ let seq = 0;+ const result = await harness.turns.runTurn({+ session: { id: `effort-${modelId}` } as HarnessTurnInput["session"],+ input: "hello",+ systemPrompt: "Reply ok.",+ history: [],+ tools: {} as HarnessTurnInput["tools"],+ scopeLabel: "personal:test",+ orgScopeId: "org:test",+ runtime: { effortLevel },+ emit: async (entry) => ({ ...entry, seq: seq++, createdAt: Date.now() }) as never,+ recordModelCall: () => {},+ cancel: AbortSignal.timeout(10_000),+ });+ assert.equal(result.reply, "ok");+ assert.equal(bodies.at(-1)?.reasoning?.effort, expected);+ });+ }+ assert.deepEqual(+ getRequiredModel(modelId).thinkingLevelMap,+ originalMap,+ "must not widen the model's supported efforts",+ );+ });+}diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.tsindex a513ca24b..2cee5fd7a 100644--- a/test/pi-harness-fast-mode.test.ts+++ b/test/pi-harness-fast-mode.test.ts@@ -7,7 +7,6 @@ import {FAST_COST_MULTIPLIER,modelSupportsFastMode,wantsFastMode,- TURN_PROVIDER_EFFORT_ALIASES,} from "../src/harness/pi-harness.ts";import { defaultInteractiveThinkingLevel } from "../src/model/pi-models.ts";@@ -69,16 +68,6 @@ test("applyFastSpeed never throws on non-object payloads", () => {assert.doesNotThrow(() => applyFastSpeed("raw", true));});-test("TURN_PROVIDER_EFFORT_ALIASES maps web-ui aliases to Anthropic effort values", () => {- assert.equal(TURN_PROVIDER_EFFORT_ALIASES.max, "max");- assert.equal(- TURN_PROVIDER_EFFORT_ALIASES.ultracode,- "max",- "Ultracode is a UI alias, not a provider effort enum (#312)",- );- assert.equal(TURN_PROVIDER_EFFORT_ALIASES.auto, null, "auto leaves effort to the provider/default path");-});-test("defaultInteractiveThinkingLevel keeps human turns light by provider", () => {assert.equal(defaultInteractiveThinkingLevel({ provider: "anthropic", api: "anthropic-messages" }), "low");assert.equal(defaultInteractiveThinkingLevel({ provider: "openai", api: "openai-responses" }), "auto");diff --git a/test/pi-harness-output-guard.test.ts b/test/pi-harness-output-guard.test.tsindex 168bc307d..cd87e312d 100644--- a/test/pi-harness-output-guard.test.ts+++ b/test/pi-harness-output-guard.test.ts@@ -187,3 +187,13 @@ test("pi-ai characterization: a stale usage anchor clamps max_tokens to 1 even fconst clamped = clampMaxTokensToContext({ contextWindow: 200_000 } as never, context as never, 64_000);assert.equal(clamped, 1, "upstream floors the cap at 1 instead of failing or compacting");});++for (const capKey of ["max_tokens", "max_output_tokens", "max_completion_tokens"]) {+ test(`output guard handles small and exhausted budgets via ${capKey}`, () => {+ const p = { [capKey]: 1, messages: [{ role: "user", content: "hello" }] };+ assert.equal(guardOutputBudget(p, FABLE).kind, "raised");+ assert.equal(p[capKey], FABLE.maxTokens);+ const full = { [capKey]: 1, messages: [{ role: "user", content: "x".repeat(790_000) }] };+ assert.throws(() => guardOutputBudget(full, FABLE), /prompt is too long/i);+ });+}