Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

fix: preserve model capabilities when mapping request options (#1395)

Joshua France1519cdc0553f
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index fb5373adb..68e9e1e96 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -17,6 +17,7 @@ import {
type Api,
type Context,
type Model,
+ type ModelThinkingLevel,
type ModelsApiStreamOptions,
type ModelsSimpleStreamOptions,
type ProviderHeaders,
@@ -24,7 +25,6 @@ import {
import { baseModelProviders, CONFIG_DEFAULTS, type Config } from "../config.ts";
type LegacyThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh";
-const LEGACY_THINKING_LEVELS = new Set<string>(["off", "minimal", "low", "medium", "high", "xhigh"]);
const TURN_EFFORT_LEVELS = new Set<string>([
"off",
"minimal",
@@ -1356,12 +1356,6 @@ export function wantsFastMode(fastMode: boolean | undefined, modelId: string | u
return fastMode === true && modelSupportsFastMode(modelId);
}
-export const TURN_PROVIDER_EFFORT_ALIASES: Record<string, string | null> = {
- max: "max",
- ultracode: "max",
- auto: null,
-};
-
export function applyFastSpeed<T>(payload: T, fast: boolean | undefined, api?: string): T {
if (fast && payload && typeof payload === "object") {
if (api && api.toLowerCase().startsWith("openai")) {
@@ -1406,9 +1400,10 @@ function estimatePayloadTokens(payload: Record<string, unknown>): number | undef
export function guardOutputBudget(payload: unknown, model: unknown): OutputBudgetGuardResult {
const p = payload as Record<string, unknown> | null;
if (!p || typeof p !== "object") return { kind: "ok" };
- let capKey: "max_tokens" | "max_output_tokens" | undefined;
+ let capKey: "max_tokens" | "max_output_tokens" | "max_completion_tokens" | undefined;
if (typeof p.max_tokens === "number") capKey = "max_tokens";
else if (typeof p.max_output_tokens === "number") capKey = "max_output_tokens";
+ else if (typeof p.max_completion_tokens === "number") capKey = "max_completion_tokens";
if (capKey === undefined) return { kind: "ok" };
const cap = p[capKey] as number;
if (cap >= OUTPUT_BUDGET_FLOOR_TOKENS) return { kind: "ok" };
@@ -1454,30 +1449,15 @@ function withFastModeHeaders(model: Model<Api>): Model<Api> {
};
}
-function applyEffortAliases(model: unknown): void {
- const mutable = model as { thinkingLevelMap?: Record<string, string | null> } | undefined;
- if (!mutable) return;
- mutable.thinkingLevelMap = {
- ...mutable.thinkingLevelMap,
- ...TURN_PROVIDER_EFFORT_ALIASES,
- };
-}
-
export function applyTurnEffort(session: AgentSession, level?: string): void {
if (!level || !TURN_EFFORT_LEVELS.has(level)) return;
const effectiveLevel =
level === "auto" && session.state.model ? defaultInteractiveThinkingLevel(session.state.model) : level;
const normalizedLevel = effectiveLevel === "auto" ? "medium" : effectiveLevel;
- applyEffortAliases(session.state.model);
- try {
- if (LEGACY_THINKING_LEVELS.has(normalizedLevel)) {
- session.setThinkingLevel(normalizedLevel as LegacyThinkingLevel);
- } else {
- session.state.thinkingLevel = normalizedLevel as typeof session.state.thinkingLevel;
- }
- } catch (e) {
- swallow("pi: set thinking level", e);
- }
+ const providerLevel = normalizedLevel === "ultracode" ? "max" : normalizedLevel;
+ // Normalize UI aliases before Pi clamps to the model's declared capabilities.
+ // Mutating thinkingLevelMap would enable efforts the provider explicitly excludes.
+ session.setThinkingLevel(providerLevel as ModelThinkingLevel);
}
export function createPiHarness(opts?: PiHarnessOptions): Harness {
diff --git a/src/model/custom-providers.ts b/src/model/custom-providers.ts
index 843211dd9..758a8ce27 100644
--- a/src/model/custom-providers.ts
+++ b/src/model/custom-providers.ts
@@ -69,6 +69,12 @@ export function validateCustomProviderSpec(spec: CustomProviderSpec): void {
for (const [field, v] of [
["contextWindow", m.contextWindow],
["maxTokens", m.maxTokens],
+ ] as const) {
+ if (v !== undefined && (!Number.isSafeInteger(v) || v <= 0)) {
+ throw new Error(`model "${m.id}": ${field} must be a positive safe integer`);
+ }
+ }
+ for (const [field, v] of [
["input", m.input],
["output", m.output],
] as const) {
diff --git a/test/custom-providers.test.ts b/test/custom-providers.test.ts
index dfe3726fa..afcbe264b 100644
--- a/test/custom-providers.test.ts
+++ b/test/custom-providers.test.ts
@@ -206,3 +206,21 @@ test("catalog cache invalidates immediately when the custom registry changes", a
const cleared = await selectableModelCatalog(fetcher);
assert.ok(!cleared.some((m) => m.id === "fresh-model"), "removal visible immediately too");
});
+
+for (const field of ["contextWindow", "maxTokens"] as const) {
+ test(`custom provider ${field} must be a positive safe integer`, () => {
+ for (const value of [0, -1, 2048.5, Number.MAX_SAFE_INTEGER + 1, NaN, Infinity]) {
+ assert.throws(
+ () => validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", [field]: value }] }),
+ /positive safe integer/,
+ `${field}=${value}`,
+ );
+ }
+ assert.doesNotThrow(() =>
+ validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", [field]: 8192 }] }),
+ );
+ assert.doesNotThrow(() =>
+ validateCustomProviderSpec({ ...GATEWAY, models: [{ id: "acme-large", input: 0.25, output: 0 }] }),
+ );
+ });
+}
diff --git a/test/pi-harness-effort-wire.test.ts b/test/pi-harness-effort-wire.test.ts
new file mode 100644
index 000000000..2d8fb1625
--- /dev/null
+++ b/test/pi-harness-effort-wire.test.ts
@@ -0,0 +1,110 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createServer } from "node:http";
+import { createPiHarness } from "../src/harness/pi-harness.ts";
+import { getRequiredModel } from "../src/model/pi-models.ts";
+import type { HarnessTurnInput } from "../src/harness/harness.ts";
+
+// Exercise the real AgentSession and SDK serializer, not just the effort setter.
+for (const [modelId, upper, allowed] of [
+ ["gpt-5.4", "xhigh", ["none", "low", "medium", "high", "xhigh"]],
+ ["gpt-5", "high", ["minimal", "low", "medium", "high"]],
+] as const) {
+ test(`Pi sends supported reasoning through HTTP for ${modelId}`, async (t) => {
+ const bodies: Array<{ reasoning?: { effort?: string } }> = [];
+ const server = createServer(async (req, res) => {
+ let raw = "";
+ for await (const chunk of req) raw += String(chunk);
+ const body = JSON.parse(raw);
+ bodies.push(body);
+ if (!(allowed as readonly string[]).includes(body.reasoning?.effort)) {
+ res.writeHead(400, { "content-type": "application/json" });
+ res.end(JSON.stringify({ error: { message: "Unsupported reasoning.effort", type: "invalid_request_error" } }));
+ return;
+ }
+ res.writeHead(200, { "content-type": "text/event-stream" });
+ const message = {
+ id: "msg_test",
+ type: "message",
+ role: "assistant",
+ status: "completed",
+ content: [{ type: "output_text", text: "ok", annotations: [] }],
+ };
+ for (const event of [
+ { type: "response.created", response: { id: "resp_test", model: modelId, status: "in_progress", output: [] } },
+ { type: "response.output_item.added", output_index: 0, item: { ...message, content: [] } },
+ {
+ type: "response.content_part.added",
+ output_index: 0,
+ content_index: 0,
+ part: { type: "output_text", text: "", annotations: [] },
+ },
+ { type: "response.output_text.delta", output_index: 0, content_index: 0, delta: "ok" },
+ { type: "response.output_item.done", output_index: 0, item: message },
+ {
+ type: "response.completed",
+ response: {
+ id: "resp_test",
+ model: modelId,
+ status: "completed",
+ output: [message],
+ usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
+ },
+ },
+ ])
+ res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
+ res.end();
+ });
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ t.after(() => new Promise<void>((resolve, reject) => server.close((e) => (e ? reject(e) : resolve()))));
+ const address = server.address();
+ assert(address && typeof address !== "string");
+ const originalMap = structuredClone(getRequiredModel(modelId).thinkingLevelMap);
+ const harness = createPiHarness({
+ defaultModelId: modelId,
+ modelGateway: {
+ url: `http://127.0.0.1:${address.port}`,
+ apiKey: "test-key",
+ apiKeyHeader: "api-key",
+ models: { [modelId]: modelId },
+ },
+ });
+ t.after(() => harness.turns.close?.());
+ // Repeat turns in the same conversation with different effort selections.
+ for (const [effortLevel, expected] of [
+ ["ultracode", upper],
+ ["max", upper],
+ ["medium", "medium"],
+ ["low", "low"],
+ ["high", "high"],
+ ["minimal", modelId === "gpt-5" ? "minimal" : "low"],
+ ["off", modelId === "gpt-5" ? "minimal" : "none"],
+ ["xhigh", upper],
+ ["auto", "medium"],
+ ]) {
+ await t.test(effortLevel!, async () => {
+ let seq = 0;
+ const result = await harness.turns.runTurn({
+ session: { id: `effort-${modelId}` } as HarnessTurnInput["session"],
+ input: "hello",
+ systemPrompt: "Reply ok.",
+ history: [],
+ tools: {} as HarnessTurnInput["tools"],
+ scopeLabel: "personal:test",
+ orgScopeId: "org:test",
+ runtime: { effortLevel },
+ emit: async (entry) => ({ ...entry, seq: seq++, createdAt: Date.now() }) as never,
+ recordModelCall: () => {},
+ cancel: AbortSignal.timeout(10_000),
+ });
+ assert.equal(result.reply, "ok");
+ assert.equal(bodies.at(-1)?.reasoning?.effort, expected);
+ });
+ }
+ assert.deepEqual(
+ getRequiredModel(modelId).thinkingLevelMap,
+ originalMap,
+ "must not widen the model's supported efforts",
+ );
+ });
+}
diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.ts
index a513ca24b..2cee5fd7a 100644
--- a/test/pi-harness-fast-mode.test.ts
+++ b/test/pi-harness-fast-mode.test.ts
@@ -7,7 +7,6 @@ import {
FAST_COST_MULTIPLIER,
modelSupportsFastMode,
wantsFastMode,
- TURN_PROVIDER_EFFORT_ALIASES,
} from "../src/harness/pi-harness.ts";
import { defaultInteractiveThinkingLevel } from "../src/model/pi-models.ts";
@@ -69,16 +68,6 @@ test("applyFastSpeed never throws on non-object payloads", () => {
assert.doesNotThrow(() => applyFastSpeed("raw", true));
});
-test("TURN_PROVIDER_EFFORT_ALIASES maps web-ui aliases to Anthropic effort values", () => {
- assert.equal(TURN_PROVIDER_EFFORT_ALIASES.max, "max");
- assert.equal(
- TURN_PROVIDER_EFFORT_ALIASES.ultracode,
- "max",
- "Ultracode is a UI alias, not a provider effort enum (#312)",
- );
- assert.equal(TURN_PROVIDER_EFFORT_ALIASES.auto, null, "auto leaves effort to the provider/default path");
-});
-
test("defaultInteractiveThinkingLevel keeps human turns light by provider", () => {
assert.equal(defaultInteractiveThinkingLevel({ provider: "anthropic", api: "anthropic-messages" }), "low");
assert.equal(defaultInteractiveThinkingLevel({ provider: "openai", api: "openai-responses" }), "auto");
diff --git a/test/pi-harness-output-guard.test.ts b/test/pi-harness-output-guard.test.ts
index 168bc307d..cd87e312d 100644
--- a/test/pi-harness-output-guard.test.ts
+++ b/test/pi-harness-output-guard.test.ts
@@ -187,3 +187,13 @@ test("pi-ai characterization: a stale usage anchor clamps max_tokens to 1 even f
const clamped = clampMaxTokensToContext({ contextWindow: 200_000 } as never, context as never, 64_000);
assert.equal(clamped, 1, "upstream floors the cap at 1 instead of failing or compacting");
});
+
+for (const capKey of ["max_tokens", "max_output_tokens", "max_completion_tokens"]) {
+ test(`output guard handles small and exhausted budgets via ${capKey}`, () => {
+ const p = { [capKey]: 1, messages: [{ role: "user", content: "hello" }] };
+ assert.equal(guardOutputBudget(p, FABLE).kind, "raised");
+ assert.equal(p[capKey], FABLE.maxTokens);
+ const full = { [capKey]: 1, messages: [{ role: "user", content: "x".repeat(790_000) }] };
+ assert.throws(() => guardOutputBudget(full, FABLE), /prompt is too long/i);
+ });
+}