Multiplayer agent harness for work.
Fall back on refusals and unavailable gateway models (#1743)
Joshua Francee33bee7a7614
diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.htmlindex f184f33d9..d1144ca88 100644--- a/plugins/admin/public/index.html+++ b/plugins/admin/public/index.html@@ -6664,6 +6664,7 @@$("card-base-model"),$("card-cron-runtime"),$("card-subagent-runtime"),+ $("card-fallback-runtime"),providers,$("card-webui-models"),$("card-soul"),@@ -8162,6 +8163,7 @@governanceUI.settings.load(r.data, scope, "runtime");governanceUI.settings.load(r.data, scope, "cron-runtime");governanceUI.settings.load(r.data, scope, "subagent-runtime");+ governanceUI.settings.load(r.data, scope, "fallback-runtime");governanceUI.settings.load(r.data, scope, "webui-models");}refreshModelChoices.push(() => governanceUI.settings.updateCatalog(r.data));@@ -8666,6 +8668,7 @@runtime: "st-runtime","cron-runtime": "st-cron-runtime","subagent-runtime": "st-subagent-runtime",+ "fallback-runtime": "st-fallback-runtime","webui-models": "st-webui-models","ack-emoji": "st-ack-emoji",branding: "st-branding",@@ -8678,6 +8681,7 @@"runtime","cron-runtime","subagent-runtime",+ "fallback-runtime","webui-models","ack-emoji","branding",diff --git a/plugins/admin/test/settings-lit.test.ts b/plugins/admin/test/settings-lit.test.tsindex 5a4e7f34f..75afb3b50 100644--- a/plugins/admin/test/settings-lit.test.ts+++ b/plugins/admin/test/settings-lit.test.ts@@ -208,10 +208,12 @@ test("purpose runtime cards independently set and clear overrides using the runtconst dom = setup();try {dom.window.eval(- "settingsUI.load(" + JSON.stringify({ ...models, cronRuntime: null, subagentRuntime: null }) + ',"org:test")',+ "settingsUI.load(" ++ JSON.stringify({ ...models, cronRuntime: null, subagentRuntime: null, fallbackRuntime: null }) ++ ',"org:test")',);const doc = dom.window.document;- for (const key of ["cron-runtime", "subagent-runtime"]) {+ for (const key of ["cron-runtime", "subagent-runtime", "fallback-runtime"]) {const collect = () => JSON.parse(String(dom.window.eval(`JSON.stringify(settingsUI.collect("${key}"))`)));assert.deepEqual(collect(), { inherit: true });assert.equal(doc.querySelectorAll(`#card-${key}`).length, 1);@@ -232,6 +234,7 @@ test("purpose runtime cards independently set and clear overrides using the runt...models,cronRuntime: { harnessId: "codex", modelId: "b", effortLevel: "low", fastMode: false },subagentRuntime: null,+ fallbackRuntime: null,}) +',"org:test")',);diff --git a/plugins/admin/test/settings-load.test.ts b/plugins/admin/test/settings-load.test.tsindex 6030c8333..558eb9073 100644--- a/plugins/admin/test/settings-load.test.ts+++ b/plugins/admin/test/settings-load.test.ts@@ -312,6 +312,7 @@ test("models view loads and saves purpose cards through the real page handlers",runtime: null,cronRuntime: null,subagentRuntime: null,+ fallbackRuntime: null,};const requests: any[] = [];const context = vm.createContext({@@ -336,7 +337,7 @@ test("models view loads and saves purpose cards through the real page handlers",extract('document.querySelectorAll("[data-save]").forEach', '$("view-governance").addEventListener("input"'),context,);- for (const key of ["cron-runtime", "subagent-runtime"]) {+ for (const key of ["cron-runtime", "subagent-runtime", "fallback-runtime"]) {assert.equal(f.document.getElementById(`card-${key}`)!.classList.contains("hidden"), false);const button = f.document.querySelector<HTMLButtonElement>(`[data-save="${key}"]`)!;(f.document.getElementById(`${key}-inherit`) as HTMLInputElement).click();diff --git a/plugins/admin/ui/settings.ts b/plugins/admin/ui/settings.tsindex 1b42d974a..1f4bf5c75 100644--- a/plugins/admin/ui/settings.ts+++ b/plugins/admin/ui/settings.ts@@ -21,7 +21,7 @@ import { repeat } from "lit/directives/repeat.js";type Model = { id: string; name?: string; effortLevels?: string[] };type Data = Record<string, any>;-const runtimeKeys = ["runtime", "cron-runtime", "subagent-runtime"];+const runtimeKeys = ["runtime", "cron-runtime", "subagent-runtime", "fallback-runtime"];const runtimeReadKey = (key: string) => key.replace(/-runtime$/, "Runtime");export class SettingsState extends SettingState {context: Data = {};@@ -187,6 +187,10 @@ function card(s: SettingsState) {runtime: ["Conversation runtime", "The default runtime for conversations unless overridden."],"cron-runtime": ["Cron runtime", "Used for scheduled jobs without a per-job runtime override."],"subagent-runtime": ["Sub-agent runtime", "Used for new sub-agents unless explicitly overridden."],+ "fallback-runtime": [+ "Fallback runtime",+ "Retries a turn or compaction when the model refuses it or is unavailable.",+ ],}[s.key]!;return html`<sectionclass=${classMap({ card: true, "sv-models": true, hidden: !s.available, dirty: s.dirty })}@@ -210,7 +214,13 @@ function card(s: SettingsState) {<span class="setting-copy"><strong>Use existing fallback</strong><small- >${s.key === "cron-runtime" ? "Use the job's scope default." : "Use the child’s scope default."}</small+ >${+ {+ "cron-runtime": "Use the job's scope default.",+ "subagent-runtime": "Use the child’s scope default.",+ "fallback-runtime": "Use the built-in Claude fallback.",+ }[s.key]+ }</small></span></label>`diff --git a/src/api/routes/admin-resources.ts b/src/api/routes/admin-resources.tsindex 1cac287a8..0a26a0089 100644--- a/src/api/routes/admin-resources.ts+++ b/src/api/routes/admin-resources.ts@@ -623,7 +623,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [return { ok: true };},},- ...([undefined, "cron", "subagent"] as const).map((purpose): AdminResource => ({+ ...([undefined, "cron", "subagent", "fallback"] as const).map((purpose): AdminResource => ({id: purpose ? `${purpose}-runtime` : "runtime",kind: "custom",target: purpose ? "org" : "any",diff --git a/src/api/routes/admin/scope-config.ts b/src/api/routes/admin/scope-config.tsindex c45b8baf9..656fb265d 100644--- a/src/api/routes/admin/scope-config.ts+++ b/src/api/routes/admin/scope-config.ts@@ -52,6 +52,7 @@ export async function putScopeConfig(ctx: ApiCtx): Promise<void> {"runtime","cron-runtime","subagent-runtime",+ "fallback-runtime","webui-models","browse-model","auto-flagger",@@ -273,6 +274,7 @@ const SETTINGS_RESOURCES = {"runtime","cronRuntime","subagentRuntime",+ "fallbackRuntime","approvedHarnesses","webuiModels","interactiveFastMode",@@ -373,7 +375,7 @@ async function scopeModelOptions(deps: ApiCtx["deps"], values: Record<string, unname: resolvedCurrent?.name ?? currentId + " (configured)",provider: resolvedCurrent?.provider ?? (currentId.includes("/") ? "openrouter" : ""),};- const purposeRuntimes = [values.cronRuntime, values.subagentRuntime].filter(+ const purposeRuntimes = [values.cronRuntime, values.subagentRuntime, values.fallbackRuntime].filter((value): value is { harnessId: string; modelId: string } =>!!value &&typeof (value as { harnessId?: unknown }).harnessId === "string" &&diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.tsindex 786d4695b..23774ee52 100644--- a/src/harness/pi-harness.ts+++ b/src/harness/pi-harness.ts@@ -126,6 +126,7 @@ export interface PiHarnessOptions {modelId?: string | ((scope?: ScopeId) => string | undefined);defaultModelId?: string;resolveBaseModelId?: () => string | undefined;+ resolveFallbackRuntime?: () => { modelId: string; effortLevel?: string; fastMode?: boolean } | undefined;detectModelId?: string;titleModelId?: string;judgeModelId?: string;@@ -1001,7 +1002,7 @@ export function piTurnError(session: AssistantTextSession, thrown: unknown, mess}const PROVIDER_REFUSAL_PATTERN =- /violate Anthropic(?:'|’)?s (?:Terms of Service|usage policy)|reduce refusals for your users by configuring a fallback model/i;+ /violate Anthropic(?:'|’)?s (?:Terms of Service|usage policy)|under Anthropic(?:'|’)?s usage policy|refusals-and-fallback|reduce refusals for your users by configuring a fallback model|the model refused to complete the request|gateway model is unavailable/i;export function isProviderRefusal(message: string | undefined): boolean {return !!message && PROVIDER_REFUSAL_PATTERN.test(message);@@ -1018,17 +1019,22 @@ export function providerRefusalError(session: AssistantTextSession, messagesBefoexport const REFUSAL_FALLBACK_MODEL_IDS = ["claude-opus-5", "claude-sonnet-5"] as const;-export function refusalFallbackModelId(fromId: string): string | undefined {+export function refusalFallbackModelId(+ fromId: string,+ configuredId?: string,+ usable: (id: string) => boolean = () => true,+): string | undefined {+ if (configuredId && configuredId !== fromId && usable(configuredId)) return configuredId;return REFUSAL_FALLBACK_MODEL_IDS.find((id) => id !== fromId);}export function refusalFallbackNote(fromModel: string, toModel: string, refusal: string): string {return (- `[system] Your previous response was blocked by the model provider's automated content filter ` +- `before it reached the user — these blocks can fire spuriously; the user did nothing wrong. ` ++ `[system] Your previous response was blocked or failed at the model provider before it reached the user ` ++ `— these failures can fire spuriously; the user did nothing wrong. ` +`The provider's stated reason was: "${refusal}". ` +`The turn has been switched from ${fromModel} to ${toModel}. Start your reply by briefly ` +- `telling the user that ${fromModel} declined this request and why (paraphrase the provider's ` ++ `telling the user that ${fromModel} could not answer this request and why (paraphrase the provider's ` +`stated reason in plain words), and that you are answering as ${toModel} instead — then answer ` +`their message.`);@@ -2216,7 +2222,10 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {const attemptRefusalFallback = async (refusal: string): Promise<boolean> => {if (userAborted || turn.cancel?.aborted) return false;const fromId = (entry.agentSession.model as { id?: string } | undefined)?.id;- const fallbackId = fromId ? refusalFallbackModelId(fromId) : undefined;+ const configured = opts?.resolveFallbackRuntime?.();+ const fallbackId = fromId+ ? refusalFallbackModelId(fromId, configured?.modelId, (id) => !!resolveModel(id, !turn.providerKeys))+ : undefined;const fallback = fallbackId ? resolveModel(fallbackId, !turn.providerKeys) : undefined;if (!fallbackId || !fallback) return false;const capMs = raceCapMs();@@ -2224,12 +2233,14 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {console.error(`[pi] provider refusal — retrying on fallback model ${fromId} -> ${fallbackId} session=${turn.session.id}: ${refusal}`,);- const wantFast = wantsFastMode(turn.runtime?.fastMode, fallbackId);+ const custom = configured?.modelId === fallbackId ? configured : undefined;+ const wantFast = wantsFastMode(custom?.fastMode ?? turn.runtime?.fastMode, fallbackId);await entry.agentSession.setModel(withRequestHeaders(fallback, !turnModelGateway?.models[fallbackId], wantFast),);entry.ref.fast = wantFast;- entry.ref.effortLevel = turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);+ entry.ref.effortLevel =+ custom?.effortLevel ?? turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);applyTurnEffort(entry.agentSession, entry.ref.effortLevel);const state = entry.agentSession.agent.state;for (let i = state.messages.length - 1; i >= messagesBefore; i--) {@@ -2534,18 +2545,38 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {},async compactHistory(input: HarnessCompactInput): Promise<string> {- const compactModelId = resolveModelId();- const model = getRequiredModel(compactModelId);+ const primaryId = resolveModelId();const providerKeys = await resolveProviderKeys();const runtime = await buildModelRuntime(providerKeys, modelGateway);- return summarizeHistory(input.history, model, (summaryModel, context, options) => {- input.recordModelCall({- model: compactModelId,- inputTokens: countTokens(context.systemPrompt ?? "") + countTokens(JSON.stringify(context.messages)),- entryCount: input.history.length,+ const summarize = (compactModelId: string) =>+ summarizeHistory(input.history, getRequiredModel(compactModelId), (summaryModel, context, options) => {+ input.recordModelCall({+ model: compactModelId,+ inputTokens: countTokens(context.systemPrompt ?? "") + countTokens(JSON.stringify(context.messages)),+ entryCount: input.history.length,+ });+ return runtime.streamSimple(summaryModel, context, options);});- return runtime.streamSimple(summaryModel, context, options);- });+ try {+ return await summarize(primaryId);+ } catch (error) {+ const fallbackId = refusalFallbackModelId(+ primaryId,+ opts?.resolveFallbackRuntime?.()?.modelId,+ (id) => !!resolveModel(id),+ );+ if (+ !(error instanceof Error) ||+ !isProviderRefusal(error.message) ||+ !fallbackId ||+ !resolveModel(fallbackId)+ )+ throw error;+ console.error(+ `[pi] compaction failed on ${primaryId}; retrying on fallback model ${fallbackId}: ${error.message}`,+ );+ return summarize(fallbackId);+ }},contextTokenBudget(scopeLabel?: string, model?: string): number | undefined {diff --git a/src/model/gateway-catalog.ts b/src/model/gateway-catalog.tsindex 987a96f51..6b3d97005 100644--- a/src/model/gateway-catalog.ts+++ b/src/model/gateway-catalog.ts@@ -6,6 +6,14 @@ const MAX_BYTES = 2 * 1024 * 1024;const MAX_MODELS = 1_000;const TTL_MS = 5 * 60_000;const RETRY_MS = 30_000;+const STALE_MS = 15 * 60_000;++function transientFailure(error: unknown): boolean {+ const status = (error as { status?: unknown } | null)?.status;+ if (typeof status === "number") return status === 429 || status >= 500;+ const name = (error as { name?: unknown } | null)?.name;+ return name === "TimeoutError" || name === "AbortError" || error instanceof TypeError;+}function record(value: unknown): Record<string, unknown> | undefined {return value !== null && typeof value === "object" && !Array.isArray(value)@@ -94,7 +102,8 @@ async function readCatalog(signal: AbortSignal.timeout(5_000),redirect: "error",});- if (!response.ok) throw new Error(`Gateway catalog returned HTTP ${response.status}`);+ if (!response.ok)+ throw Object.assign(new Error(`Gateway catalog returned HTTP ${response.status}`), { status: response.status });if (!response.body || Number(response.headers.get("content-length")) > MAX_BYTES)throw new Error("Gateway catalog exceeds size limit");const reader = response.body.getReader();@@ -130,6 +139,7 @@ export function createGatewayCatalog(let discovered = false;let expiresAt = 0;let retryAt = 0;+ let lastSuccessAt = 0;let inFlight: Promise<void> | undefined;const transport: ModelGatewayTransportConfig = {...config,@@ -169,6 +179,7 @@ export function createGatewayCatalog(discovered = true;expiresAt = now() + TTL_MS;retryAt = expiresAt;+ lastSuccessAt = now();const offeredAliases = new Set(selectableBaseModels().filter(({ id }) => modelOfferedInWebui(id))@@ -182,8 +193,12 @@ export function createGatewayCatalog(: [],),);- } catch {+ } catch (error) {retryAt = now() + RETRY_MS;+ if (discovered && transientFailure(error) && now() - lastSuccessAt < STALE_MS) {+ console.warn("[model] Gateway model discovery failed; keeping last known models and retrying in 30 seconds");+ return;+ }if (discovered || Object.keys(config.models).length === 0) {routes = {};expiresAt = 0;diff --git a/src/resolution/config-store.ts b/src/resolution/config-store.tsindex 56915837f..cff52a6fd 100644--- a/src/resolution/config-store.ts+++ b/src/resolution/config-store.ts@@ -74,13 +74,14 @@ export interface PersistedBaseModel {modelId?: string;cronRuntime?: RuntimeSelection;subagentRuntime?: RuntimeSelection;+ fallbackRuntime?: RuntimeSelection;harnessId?: string;orgRevision?: number;revision?: number;effortLevel?: string;fastMode?: boolean;}-export type RuntimePurpose = "cron" | "subagent";+export type RuntimePurpose = "cron" | "subagent" | "fallback";interface RuntimeSelection {harnessId: string;@@ -403,6 +404,7 @@ export function createMemoryConfigStore(? {...(row?.cronRuntime ? { cronRuntime: row.cronRuntime } : {}),...(row?.subagentRuntime ? { subagentRuntime: row.subagentRuntime } : {}),+ ...(row?.fallbackRuntime ? { fallbackRuntime: row.fallbackRuntime } : {}),}: {};const clearRuntime = async (id: ScopeId, row?: PersistedBaseModel | null) => {diff --git a/src/wiring.ts b/src/wiring.tsindex 793389c6c..88db6cd4c 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -1370,6 +1370,7 @@ export function buildApp(...piHarnessConfigOptions(config),modelGateway: gatewayTransport,resolveBaseModelId: () => orgBaseModelId() ?? defaultForHarness("pi"),+ resolveFallbackRuntime: () => configStore.getPurposeRuntime("fallback"),resolveProviderKeys: resolveModelProviderKeys,signals: runSignals,mcpTools,diff --git a/test/admin-resources.test.ts b/test/admin-resources.test.tsindex c798de634..ec848efdb 100644--- a/test/admin-resources.test.ts+++ b/test/admin-resources.test.ts@@ -948,7 +948,7 @@ test("org purpose runtimes round-trip, validate, and clear independently", asynctry {assert.equal((await get()).cronRuntime, null);assert.equal((await get()).subagentRuntime, null);- for (const purpose of ["cron", "subagent"] as const) {+ for (const purpose of ["cron", "subagent", "fallback"] as const) {const resource = `${purpose}-runtime`;const choice = { harnessId: "pi", modelId: "claude-opus-5", effortLevel: "low", fastMode: false };assert.equal(diff --git a/test/gateway-catalog.test.ts b/test/gateway-catalog.test.tsindex ab0eae136..a6b0c9e88 100644--- a/test/gateway-catalog.test.ts+++ b/test/gateway-catalog.test.ts@@ -167,13 +167,21 @@ test("legacy routes work without discovery support, then discovery adds models aassert.deepEqual(f.catalog.transport.models, {});});-test("discovery outage hides stale models; later successful refresh recovers", async () => {- const f = fixture();+test("a transient discovery blip keeps last known routes; a prolonged outage hides them", async () => {+ const f = fixture([group()], { "claude-opus-5": "vendor/new-model" });await f.catalog.refresh();+ const native = resolveModel("claude-opus-5")!;f.tick();f.status(503);await f.catalog.refresh();+ assert.ok(resolveModel("gateway/vendor/new-model"));+ assert.ok(modelGatewayRequest(f.catalog.transport, native));+ for (let i = 0; i < 3; i++) {+ f.tick();+ await f.catalog.refresh();+ }assert.equal(resolveModel("gateway/vendor/new-model"), undefined);+ assert.throws(() => modelGatewayRequest(f.catalog.transport, native), /unavailable/);f.tick();f.status(200);f.metadata([group("new")]);diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.tsindex 31ac08b83..3eacfe70d 100644--- a/test/pi-harness-fast-mode.test.ts+++ b/test/pi-harness-fast-mode.test.ts@@ -12,6 +12,7 @@ import {wantsFastMode,createPiHarness,} from "../src/harness/pi-harness.ts";+import type { PiHarnessOptions } from "../src/harness/pi-harness.ts";import type { HarnessLlmRequestRecord, HarnessTurnInput } from "../src/harness/harness.ts";import type { NewEntry } from "../src/sessions/session-store.ts";import type { SessionEntry } from "../src/types.ts";@@ -313,6 +314,7 @@ async function runTurn(fastMode: boolean,respond: (payload: Record<string, unknown>, index: number) => Response,gateway = false,+ extra: Partial<PiHarnessOptions> = {},): Promise<{ rows: HarnessLlmRequestRecord[]; payloads: Array<Record<string, unknown>>; betas: Array<string | null> }> {const rows: HarnessLlmRequestRecord[] = [];const payloads: Array<Record<string, unknown>> = [];@@ -334,6 +336,7 @@ async function runTurn(},}: {}),+ ...extra,});const realFetch = globalThis.fetch;globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {@@ -470,3 +473,91 @@ for (const gateway of [true, false]) {assert.equal(rows[0]!.model, "gpt-6-astra-ultrafast");});}++const USAGE_POLICY_REFUSAL =+ "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, see https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback.";++test("a current Usage Policy refusal retries on the admin-configured fallback runtime", async () => {+ const { payloads } = await runTurn(+ "refusal-configured-fallback",+ "claude-sonnet-5",+ false,+ (_payload, index) =>+ index === 0+ ? new Response(+ JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: USAGE_POLICY_REFUSAL } }),+ {+ status: 400,+ headers: { "content-type": "application/json" },+ },+ )+ : responsesReply("recovered", ASTRA_WIRE_USAGE),+ false,+ { resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol", effortLevel: "low" }) },+ );+ assert.equal(payloads.length, 2);+ assert.equal(payloads[1]?.model, "gpt-6-sol");+ assert.equal((payloads[1]?.reasoning as { effort?: string } | undefined)?.effort, "low");+});++test("an unavailable gateway model retries on the configured fallback instead of failing the turn", async () => {+ const { payloads } = await runTurn(+ "unavailable-configured-fallback",+ "claude-sonnet-5",+ false,+ () => responsesReply("recovered", ASTRA_WIRE_USAGE),+ false,+ {+ modelGateway: {+ url: "https://gateway.example/v1",+ apiKey: "k",+ apiKeyHeader: "x-k",+ models: {},+ reservedModelIds: new Set(["claude-sonnet-5"]),+ },+ resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol", effortLevel: "low" }),+ },+ );+ assert.deepEqual(+ payloads.map((p) => p.model),+ ["gpt-6-sol"],+ );+});++test("compaction retries a refused summary on the configured fallback model", async () => {+ const harness = createPiHarness({+ apiKey: "sk-anthropic-test",+ openaiApiKey: "sk-openai-test",+ modelId: "claude-sonnet-5",+ resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol" }),+ });+ const models: unknown[] = [];+ const realFetch = globalThis.fetch;+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {+ const payload = JSON.parse(String(init?.body ?? "{}")) as Record<string, unknown>;+ models.push(payload.model);+ return models.length === 1+ ? new Response(+ JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: USAGE_POLICY_REFUSAL } }),+ {+ status: 400,+ headers: { "content-type": "application/json" },+ },+ )+ : responsesReply("summary of the work", ASTRA_WIRE_USAGE);+ }) as typeof globalThis.fetch;+ try {+ const text = await harness.models.compactHistory!({+ session: { id: "compact-fallback" } as HarnessTurnInput["session"],+ history: [+ { seq: 1, kind: "user", payload: { text: "hello" }, createdAt: 1 },+ { seq: 2, kind: "assistant", payload: { text: "hi" }, createdAt: 2 },+ ] as unknown as SessionEntry[],+ recordModelCall: () => {},+ });+ assert.match(text, /summary of the work/);+ } finally {+ globalThis.fetch = realFetch;+ }+ assert.deepEqual(models, ["claude-sonnet-5", "gpt-6-sol"]);+});diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.tsindex 7424e1b4c..1be785bb6 100644--- a/test/pi-harness-oneshot.test.ts+++ b/test/pi-harness-oneshot.test.ts@@ -921,6 +921,14 @@ test("isProviderRefusal matches Anthropic's ToS-refusal wording and nothing elsetrue,);assert.equal(isProviderRefusal("This request seems to violate Anthropic’s usage policy."), true);+ assert.equal(+ isProviderRefusal(+ "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, see https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback.",+ ),+ true,+ );+ assert.equal(isProviderRefusal("The model refused to complete the request"), true);+ assert.equal(isProviderRefusal("Gateway model is unavailable: claude-opus-5-5"), true);assert.equal(isProviderRefusal("Anthropic API error (overloaded_error): Overloaded"), false);assert.equal(isProviderRefusal("prompt is too long: 250000 tokens > 200000 maximum"), false);assert.equal(isProviderRefusal(undefined), false);@@ -958,6 +966,12 @@ test("refusal fallback drawdown: Fable -> Opus, Opus -> Sonnet, never the refuseassert.equal(refusalFallbackModelId("claude-fable-5-1"), "claude-opus-5");assert.equal(refusalFallbackModelId("claude-fable-5"), "claude-opus-5");assert.equal(refusalFallbackModelId("claude-opus-5"), "claude-sonnet-5");+ assert.equal(refusalFallbackModelId("claude-opus-5", "gpt-6-sol"), "gpt-6-sol");+ assert.equal(refusalFallbackModelId("gpt-6-sol", "gpt-6-sol"), "claude-opus-5");+ assert.equal(+ refusalFallbackModelId("claude-opus-5", "not-a-pi-model", (id) => id !== "not-a-pi-model"),+ "claude-sonnet-5",+ );for (const id of REFUSAL_FALLBACK_MODEL_IDS) {assert.notEqual(refusalFallbackModelId(id), id);assert.equal(resolveModel(id)?.provider, "anthropic");diff --git a/test/runtime-purpose.test.ts b/test/runtime-purpose.test.tsindex 4882ff570..a1d8b78bc 100644--- a/test/runtime-purpose.test.ts+++ b/test/runtime-purpose.test.ts@@ -46,7 +46,7 @@ test("cron and loop entry points defer legacy injections until runtime resolutiotest("each purpose ignores conversation overrides and explicit fields including false win", async () => {const config = setup();const conversation = resolveRuntimeChoice(config, ORG, SCOPE, fallback);- for (const purpose of ["cron", "subagent"] as const) {+ for (const purpose of ["cron", "subagent", "fallback"] as const) {await config.setPurposeRuntime(purpose, category);assert.deepEqual(resolveRuntimeChoice(config, ORG, SCOPE, fallback, undefined, purpose), category);assert.deepEqual(