Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fall back on refusals and unavailable gateway models (#1743)

Joshua Francee33bee7a7614
diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.html
index f184f33d9..d1144ca88 100644
--- a/plugins/admin/public/index.html
+++ b/plugins/admin/public/index.html
@@ -6664,6 +6664,7 @@
$("card-base-model"),
$("card-cron-runtime"),
$("card-subagent-runtime"),
+ $("card-fallback-runtime"),
providers,
$("card-webui-models"),
$("card-soul"),
@@ -8162,6 +8163,7 @@
governanceUI.settings.load(r.data, scope, "runtime");
governanceUI.settings.load(r.data, scope, "cron-runtime");
governanceUI.settings.load(r.data, scope, "subagent-runtime");
+ governanceUI.settings.load(r.data, scope, "fallback-runtime");
governanceUI.settings.load(r.data, scope, "webui-models");
}
refreshModelChoices.push(() => governanceUI.settings.updateCatalog(r.data));
@@ -8666,6 +8668,7 @@
runtime: "st-runtime",
"cron-runtime": "st-cron-runtime",
"subagent-runtime": "st-subagent-runtime",
+ "fallback-runtime": "st-fallback-runtime",
"webui-models": "st-webui-models",
"ack-emoji": "st-ack-emoji",
branding: "st-branding",
@@ -8678,6 +8681,7 @@
"runtime",
"cron-runtime",
"subagent-runtime",
+ "fallback-runtime",
"webui-models",
"ack-emoji",
"branding",
diff --git a/plugins/admin/test/settings-lit.test.ts b/plugins/admin/test/settings-lit.test.ts
index 5a4e7f34f..75afb3b50 100644
--- a/plugins/admin/test/settings-lit.test.ts
+++ b/plugins/admin/test/settings-lit.test.ts
@@ -208,10 +208,12 @@ test("purpose runtime cards independently set and clear overrides using the runt
const dom = setup();
try {
dom.window.eval(
- "settingsUI.load(" + JSON.stringify({ ...models, cronRuntime: null, subagentRuntime: null }) + ',"org:test")',
+ "settingsUI.load(" +
+ JSON.stringify({ ...models, cronRuntime: null, subagentRuntime: null, fallbackRuntime: null }) +
+ ',"org:test")',
);
const doc = dom.window.document;
- for (const key of ["cron-runtime", "subagent-runtime"]) {
+ for (const key of ["cron-runtime", "subagent-runtime", "fallback-runtime"]) {
const collect = () => JSON.parse(String(dom.window.eval(`JSON.stringify(settingsUI.collect("${key}"))`)));
assert.deepEqual(collect(), { inherit: true });
assert.equal(doc.querySelectorAll(`#card-${key}`).length, 1);
@@ -232,6 +234,7 @@ test("purpose runtime cards independently set and clear overrides using the runt
...models,
cronRuntime: { harnessId: "codex", modelId: "b", effortLevel: "low", fastMode: false },
subagentRuntime: null,
+ fallbackRuntime: null,
}) +
',"org:test")',
);
diff --git a/plugins/admin/test/settings-load.test.ts b/plugins/admin/test/settings-load.test.ts
index 6030c8333..558eb9073 100644
--- a/plugins/admin/test/settings-load.test.ts
+++ b/plugins/admin/test/settings-load.test.ts
@@ -312,6 +312,7 @@ test("models view loads and saves purpose cards through the real page handlers",
runtime: null,
cronRuntime: null,
subagentRuntime: null,
+ fallbackRuntime: null,
};
const requests: any[] = [];
const context = vm.createContext({
@@ -336,7 +337,7 @@ test("models view loads and saves purpose cards through the real page handlers",
extract('document.querySelectorAll("[data-save]").forEach', '$("view-governance").addEventListener("input"'),
context,
);
- for (const key of ["cron-runtime", "subagent-runtime"]) {
+ for (const key of ["cron-runtime", "subagent-runtime", "fallback-runtime"]) {
assert.equal(f.document.getElementById(`card-${key}`)!.classList.contains("hidden"), false);
const button = f.document.querySelector<HTMLButtonElement>(`[data-save="${key}"]`)!;
(f.document.getElementById(`${key}-inherit`) as HTMLInputElement).click();
diff --git a/plugins/admin/ui/settings.ts b/plugins/admin/ui/settings.ts
index 1b42d974a..1f4bf5c75 100644
--- a/plugins/admin/ui/settings.ts
+++ b/plugins/admin/ui/settings.ts
@@ -21,7 +21,7 @@ import { repeat } from "lit/directives/repeat.js";
type Model = { id: string; name?: string; effortLevels?: string[] };
type Data = Record<string, any>;
-const runtimeKeys = ["runtime", "cron-runtime", "subagent-runtime"];
+const runtimeKeys = ["runtime", "cron-runtime", "subagent-runtime", "fallback-runtime"];
const runtimeReadKey = (key: string) => key.replace(/-runtime$/, "Runtime");
export class SettingsState extends SettingState {
context: Data = {};
@@ -187,6 +187,10 @@ function card(s: SettingsState) {
runtime: ["Conversation runtime", "The default runtime for conversations unless overridden."],
"cron-runtime": ["Cron runtime", "Used for scheduled jobs without a per-job runtime override."],
"subagent-runtime": ["Sub-agent runtime", "Used for new sub-agents unless explicitly overridden."],
+ "fallback-runtime": [
+ "Fallback runtime",
+ "Retries a turn or compaction when the model refuses it or is unavailable.",
+ ],
}[s.key]!;
return html`<section
class=${classMap({ card: true, "sv-models": true, hidden: !s.available, dirty: s.dirty })}
@@ -210,7 +214,13 @@ function card(s: SettingsState) {
<span class="setting-copy"
><strong>Use existing fallback</strong
><small
- >${s.key === "cron-runtime" ? "Use the job's scope default." : "Use the child’s scope default."}</small
+ >${
+ {
+ "cron-runtime": "Use the job's scope default.",
+ "subagent-runtime": "Use the child’s scope default.",
+ "fallback-runtime": "Use the built-in Claude fallback.",
+ }[s.key]
+ }</small
></span
>
</label>`
diff --git a/src/api/routes/admin-resources.ts b/src/api/routes/admin-resources.ts
index 1cac287a8..0a26a0089 100644
--- a/src/api/routes/admin-resources.ts
+++ b/src/api/routes/admin-resources.ts
@@ -623,7 +623,7 @@ export const ADMIN_RESOURCES: readonly AdminResource[] = [
return { ok: true };
},
},
- ...([undefined, "cron", "subagent"] as const).map((purpose): AdminResource => ({
+ ...([undefined, "cron", "subagent", "fallback"] as const).map((purpose): AdminResource => ({
id: purpose ? `${purpose}-runtime` : "runtime",
kind: "custom",
target: purpose ? "org" : "any",
diff --git a/src/api/routes/admin/scope-config.ts b/src/api/routes/admin/scope-config.ts
index c45b8baf9..656fb265d 100644
--- a/src/api/routes/admin/scope-config.ts
+++ b/src/api/routes/admin/scope-config.ts
@@ -52,6 +52,7 @@ export async function putScopeConfig(ctx: ApiCtx): Promise<void> {
"runtime",
"cron-runtime",
"subagent-runtime",
+ "fallback-runtime",
"webui-models",
"browse-model",
"auto-flagger",
@@ -273,6 +274,7 @@ const SETTINGS_RESOURCES = {
"runtime",
"cronRuntime",
"subagentRuntime",
+ "fallbackRuntime",
"approvedHarnesses",
"webuiModels",
"interactiveFastMode",
@@ -373,7 +375,7 @@ async function scopeModelOptions(deps: ApiCtx["deps"], values: Record<string, un
name: resolvedCurrent?.name ?? currentId + " (configured)",
provider: resolvedCurrent?.provider ?? (currentId.includes("/") ? "openrouter" : ""),
};
- const purposeRuntimes = [values.cronRuntime, values.subagentRuntime].filter(
+ const purposeRuntimes = [values.cronRuntime, values.subagentRuntime, values.fallbackRuntime].filter(
(value): value is { harnessId: string; modelId: string } =>
!!value &&
typeof (value as { harnessId?: unknown }).harnessId === "string" &&
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index 786d4695b..23774ee52 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -126,6 +126,7 @@ export interface PiHarnessOptions {
modelId?: string | ((scope?: ScopeId) => string | undefined);
defaultModelId?: string;
resolveBaseModelId?: () => string | undefined;
+ resolveFallbackRuntime?: () => { modelId: string; effortLevel?: string; fastMode?: boolean } | undefined;
detectModelId?: string;
titleModelId?: string;
judgeModelId?: string;
@@ -1001,7 +1002,7 @@ export function piTurnError(session: AssistantTextSession, thrown: unknown, mess
}
const PROVIDER_REFUSAL_PATTERN =
- /violate Anthropic(?:'|’)?s (?:Terms of Service|usage policy)|reduce refusals for your users by configuring a fallback model/i;
+ /violate Anthropic(?:'|’)?s (?:Terms of Service|usage policy)|under Anthropic(?:'|’)?s usage policy|refusals-and-fallback|reduce refusals for your users by configuring a fallback model|the model refused to complete the request|gateway model is unavailable/i;
export function isProviderRefusal(message: string | undefined): boolean {
return !!message && PROVIDER_REFUSAL_PATTERN.test(message);
@@ -1018,17 +1019,22 @@ export function providerRefusalError(session: AssistantTextSession, messagesBefo
export const REFUSAL_FALLBACK_MODEL_IDS = ["claude-opus-5", "claude-sonnet-5"] as const;
-export function refusalFallbackModelId(fromId: string): string | undefined {
+export function refusalFallbackModelId(
+ fromId: string,
+ configuredId?: string,
+ usable: (id: string) => boolean = () => true,
+): string | undefined {
+ if (configuredId && configuredId !== fromId && usable(configuredId)) return configuredId;
return REFUSAL_FALLBACK_MODEL_IDS.find((id) => id !== fromId);
}
export function refusalFallbackNote(fromModel: string, toModel: string, refusal: string): string {
return (
- `[system] Your previous response was blocked by the model provider's automated content filter ` +
- `before it reached the user — these blocks can fire spuriously; the user did nothing wrong. ` +
+ `[system] Your previous response was blocked or failed at the model provider before it reached the user ` +
+ `— these failures can fire spuriously; the user did nothing wrong. ` +
`The provider's stated reason was: "${refusal}". ` +
`The turn has been switched from ${fromModel} to ${toModel}. Start your reply by briefly ` +
- `telling the user that ${fromModel} declined this request and why (paraphrase the provider's ` +
+ `telling the user that ${fromModel} could not answer this request and why (paraphrase the provider's ` +
`stated reason in plain words), and that you are answering as ${toModel} instead — then answer ` +
`their message.`
);
@@ -2216,7 +2222,10 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
const attemptRefusalFallback = async (refusal: string): Promise<boolean> => {
if (userAborted || turn.cancel?.aborted) return false;
const fromId = (entry.agentSession.model as { id?: string } | undefined)?.id;
- const fallbackId = fromId ? refusalFallbackModelId(fromId) : undefined;
+ const configured = opts?.resolveFallbackRuntime?.();
+ const fallbackId = fromId
+ ? refusalFallbackModelId(fromId, configured?.modelId, (id) => !!resolveModel(id, !turn.providerKeys))
+ : undefined;
const fallback = fallbackId ? resolveModel(fallbackId, !turn.providerKeys) : undefined;
if (!fallbackId || !fallback) return false;
const capMs = raceCapMs();
@@ -2224,12 +2233,14 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
console.error(
`[pi] provider refusal — retrying on fallback model ${fromId} -> ${fallbackId} session=${turn.session.id}: ${refusal}`,
);
- const wantFast = wantsFastMode(turn.runtime?.fastMode, fallbackId);
+ const custom = configured?.modelId === fallbackId ? configured : undefined;
+ const wantFast = wantsFastMode(custom?.fastMode ?? turn.runtime?.fastMode, fallbackId);
await entry.agentSession.setModel(
withRequestHeaders(fallback, !turnModelGateway?.models[fallbackId], wantFast),
);
entry.ref.fast = wantFast;
- entry.ref.effortLevel = turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);
+ entry.ref.effortLevel =
+ custom?.effortLevel ?? turn.runtime?.effortLevel ?? defaultInteractiveThinkingLevel(fallback);
applyTurnEffort(entry.agentSession, entry.ref.effortLevel);
const state = entry.agentSession.agent.state;
for (let i = state.messages.length - 1; i >= messagesBefore; i--) {
@@ -2534,18 +2545,38 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
},
async compactHistory(input: HarnessCompactInput): Promise<string> {
- const compactModelId = resolveModelId();
- const model = getRequiredModel(compactModelId);
+ const primaryId = resolveModelId();
const providerKeys = await resolveProviderKeys();
const runtime = await buildModelRuntime(providerKeys, modelGateway);
- return summarizeHistory(input.history, model, (summaryModel, context, options) => {
- input.recordModelCall({
- model: compactModelId,
- inputTokens: countTokens(context.systemPrompt ?? "") + countTokens(JSON.stringify(context.messages)),
- entryCount: input.history.length,
+ const summarize = (compactModelId: string) =>
+ summarizeHistory(input.history, getRequiredModel(compactModelId), (summaryModel, context, options) => {
+ input.recordModelCall({
+ model: compactModelId,
+ inputTokens: countTokens(context.systemPrompt ?? "") + countTokens(JSON.stringify(context.messages)),
+ entryCount: input.history.length,
+ });
+ return runtime.streamSimple(summaryModel, context, options);
});
- return runtime.streamSimple(summaryModel, context, options);
- });
+ try {
+ return await summarize(primaryId);
+ } catch (error) {
+ const fallbackId = refusalFallbackModelId(
+ primaryId,
+ opts?.resolveFallbackRuntime?.()?.modelId,
+ (id) => !!resolveModel(id),
+ );
+ if (
+ !(error instanceof Error) ||
+ !isProviderRefusal(error.message) ||
+ !fallbackId ||
+ !resolveModel(fallbackId)
+ )
+ throw error;
+ console.error(
+ `[pi] compaction failed on ${primaryId}; retrying on fallback model ${fallbackId}: ${error.message}`,
+ );
+ return summarize(fallbackId);
+ }
},
contextTokenBudget(scopeLabel?: string, model?: string): number | undefined {
diff --git a/src/model/gateway-catalog.ts b/src/model/gateway-catalog.ts
index 987a96f51..6b3d97005 100644
--- a/src/model/gateway-catalog.ts
+++ b/src/model/gateway-catalog.ts
@@ -6,6 +6,14 @@ const MAX_BYTES = 2 * 1024 * 1024;
const MAX_MODELS = 1_000;
const TTL_MS = 5 * 60_000;
const RETRY_MS = 30_000;
+const STALE_MS = 15 * 60_000;
+
+function transientFailure(error: unknown): boolean {
+ const status = (error as { status?: unknown } | null)?.status;
+ if (typeof status === "number") return status === 429 || status >= 500;
+ const name = (error as { name?: unknown } | null)?.name;
+ return name === "TimeoutError" || name === "AbortError" || error instanceof TypeError;
+}
function record(value: unknown): Record<string, unknown> | undefined {
return value !== null && typeof value === "object" && !Array.isArray(value)
@@ -94,7 +102,8 @@ async function readCatalog(
signal: AbortSignal.timeout(5_000),
redirect: "error",
});
- if (!response.ok) throw new Error(`Gateway catalog returned HTTP ${response.status}`);
+ if (!response.ok)
+ throw Object.assign(new Error(`Gateway catalog returned HTTP ${response.status}`), { status: response.status });
if (!response.body || Number(response.headers.get("content-length")) > MAX_BYTES)
throw new Error("Gateway catalog exceeds size limit");
const reader = response.body.getReader();
@@ -130,6 +139,7 @@ export function createGatewayCatalog(
let discovered = false;
let expiresAt = 0;
let retryAt = 0;
+ let lastSuccessAt = 0;
let inFlight: Promise<void> | undefined;
const transport: ModelGatewayTransportConfig = {
...config,
@@ -169,6 +179,7 @@ export function createGatewayCatalog(
discovered = true;
expiresAt = now() + TTL_MS;
retryAt = expiresAt;
+ lastSuccessAt = now();
const offeredAliases = new Set(
selectableBaseModels()
.filter(({ id }) => modelOfferedInWebui(id))
@@ -182,8 +193,12 @@ export function createGatewayCatalog(
: [],
),
);
- } catch {
+ } catch (error) {
retryAt = now() + RETRY_MS;
+ if (discovered && transientFailure(error) && now() - lastSuccessAt < STALE_MS) {
+ console.warn("[model] Gateway model discovery failed; keeping last known models and retrying in 30 seconds");
+ return;
+ }
if (discovered || Object.keys(config.models).length === 0) {
routes = {};
expiresAt = 0;
diff --git a/src/resolution/config-store.ts b/src/resolution/config-store.ts
index 56915837f..cff52a6fd 100644
--- a/src/resolution/config-store.ts
+++ b/src/resolution/config-store.ts
@@ -74,13 +74,14 @@ export interface PersistedBaseModel {
modelId?: string;
cronRuntime?: RuntimeSelection;
subagentRuntime?: RuntimeSelection;
+ fallbackRuntime?: RuntimeSelection;
harnessId?: string;
orgRevision?: number;
revision?: number;
effortLevel?: string;
fastMode?: boolean;
}
-export type RuntimePurpose = "cron" | "subagent";
+export type RuntimePurpose = "cron" | "subagent" | "fallback";
interface RuntimeSelection {
harnessId: string;
@@ -403,6 +404,7 @@ export function createMemoryConfigStore(
? {
...(row?.cronRuntime ? { cronRuntime: row.cronRuntime } : {}),
...(row?.subagentRuntime ? { subagentRuntime: row.subagentRuntime } : {}),
+ ...(row?.fallbackRuntime ? { fallbackRuntime: row.fallbackRuntime } : {}),
}
: {};
const clearRuntime = async (id: ScopeId, row?: PersistedBaseModel | null) => {
diff --git a/src/wiring.ts b/src/wiring.ts
index 793389c6c..88db6cd4c 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -1370,6 +1370,7 @@ export function buildApp(
...piHarnessConfigOptions(config),
modelGateway: gatewayTransport,
resolveBaseModelId: () => orgBaseModelId() ?? defaultForHarness("pi"),
+ resolveFallbackRuntime: () => configStore.getPurposeRuntime("fallback"),
resolveProviderKeys: resolveModelProviderKeys,
signals: runSignals,
mcpTools,
diff --git a/test/admin-resources.test.ts b/test/admin-resources.test.ts
index c798de634..ec848efdb 100644
--- a/test/admin-resources.test.ts
+++ b/test/admin-resources.test.ts
@@ -948,7 +948,7 @@ test("org purpose runtimes round-trip, validate, and clear independently", async
try {
assert.equal((await get()).cronRuntime, null);
assert.equal((await get()).subagentRuntime, null);
- for (const purpose of ["cron", "subagent"] as const) {
+ for (const purpose of ["cron", "subagent", "fallback"] as const) {
const resource = `${purpose}-runtime`;
const choice = { harnessId: "pi", modelId: "claude-opus-5", effortLevel: "low", fastMode: false };
assert.equal(
diff --git a/test/gateway-catalog.test.ts b/test/gateway-catalog.test.ts
index ab0eae136..a6b0c9e88 100644
--- a/test/gateway-catalog.test.ts
+++ b/test/gateway-catalog.test.ts
@@ -167,13 +167,21 @@ test("legacy routes work without discovery support, then discovery adds models a
assert.deepEqual(f.catalog.transport.models, {});
});
-test("discovery outage hides stale models; later successful refresh recovers", async () => {
- const f = fixture();
+test("a transient discovery blip keeps last known routes; a prolonged outage hides them", async () => {
+ const f = fixture([group()], { "claude-opus-5": "vendor/new-model" });
await f.catalog.refresh();
+ const native = resolveModel("claude-opus-5")!;
f.tick();
f.status(503);
await f.catalog.refresh();
+ assert.ok(resolveModel("gateway/vendor/new-model"));
+ assert.ok(modelGatewayRequest(f.catalog.transport, native));
+ for (let i = 0; i < 3; i++) {
+ f.tick();
+ await f.catalog.refresh();
+ }
assert.equal(resolveModel("gateway/vendor/new-model"), undefined);
+ assert.throws(() => modelGatewayRequest(f.catalog.transport, native), /unavailable/);
f.tick();
f.status(200);
f.metadata([group("new")]);
diff --git a/test/pi-harness-fast-mode.test.ts b/test/pi-harness-fast-mode.test.ts
index 31ac08b83..3eacfe70d 100644
--- a/test/pi-harness-fast-mode.test.ts
+++ b/test/pi-harness-fast-mode.test.ts
@@ -12,6 +12,7 @@ import {
wantsFastMode,
createPiHarness,
} from "../src/harness/pi-harness.ts";
+import type { PiHarnessOptions } from "../src/harness/pi-harness.ts";
import type { HarnessLlmRequestRecord, HarnessTurnInput } from "../src/harness/harness.ts";
import type { NewEntry } from "../src/sessions/session-store.ts";
import type { SessionEntry } from "../src/types.ts";
@@ -313,6 +314,7 @@ async function runTurn(
fastMode: boolean,
respond: (payload: Record<string, unknown>, index: number) => Response,
gateway = false,
+ extra: Partial<PiHarnessOptions> = {},
): Promise<{ rows: HarnessLlmRequestRecord[]; payloads: Array<Record<string, unknown>>; betas: Array<string | null> }> {
const rows: HarnessLlmRequestRecord[] = [];
const payloads: Array<Record<string, unknown>> = [];
@@ -334,6 +336,7 @@ async function runTurn(
},
}
: {}),
+ ...extra,
});
const realFetch = globalThis.fetch;
globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
@@ -470,3 +473,91 @@ for (const gateway of [true, false]) {
assert.equal(rows[0]!.model, "gpt-6-astra-ultrafast");
});
}
+
+const USAGE_POLICY_REFUSAL =
+ "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, see https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback.";
+
+test("a current Usage Policy refusal retries on the admin-configured fallback runtime", async () => {
+ const { payloads } = await runTurn(
+ "refusal-configured-fallback",
+ "claude-sonnet-5",
+ false,
+ (_payload, index) =>
+ index === 0
+ ? new Response(
+ JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: USAGE_POLICY_REFUSAL } }),
+ {
+ status: 400,
+ headers: { "content-type": "application/json" },
+ },
+ )
+ : responsesReply("recovered", ASTRA_WIRE_USAGE),
+ false,
+ { resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol", effortLevel: "low" }) },
+ );
+ assert.equal(payloads.length, 2);
+ assert.equal(payloads[1]?.model, "gpt-6-sol");
+ assert.equal((payloads[1]?.reasoning as { effort?: string } | undefined)?.effort, "low");
+});
+
+test("an unavailable gateway model retries on the configured fallback instead of failing the turn", async () => {
+ const { payloads } = await runTurn(
+ "unavailable-configured-fallback",
+ "claude-sonnet-5",
+ false,
+ () => responsesReply("recovered", ASTRA_WIRE_USAGE),
+ false,
+ {
+ modelGateway: {
+ url: "https://gateway.example/v1",
+ apiKey: "k",
+ apiKeyHeader: "x-k",
+ models: {},
+ reservedModelIds: new Set(["claude-sonnet-5"]),
+ },
+ resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol", effortLevel: "low" }),
+ },
+ );
+ assert.deepEqual(
+ payloads.map((p) => p.model),
+ ["gpt-6-sol"],
+ );
+});
+
+test("compaction retries a refused summary on the configured fallback model", async () => {
+ const harness = createPiHarness({
+ apiKey: "sk-anthropic-test",
+ openaiApiKey: "sk-openai-test",
+ modelId: "claude-sonnet-5",
+ resolveFallbackRuntime: () => ({ modelId: "gpt-6-sol" }),
+ });
+ const models: unknown[] = [];
+ const realFetch = globalThis.fetch;
+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+ const payload = JSON.parse(String(init?.body ?? "{}")) as Record<string, unknown>;
+ models.push(payload.model);
+ return models.length === 1
+ ? new Response(
+ JSON.stringify({ type: "error", error: { type: "invalid_request_error", message: USAGE_POLICY_REFUSAL } }),
+ {
+ status: 400,
+ headers: { "content-type": "application/json" },
+ },
+ )
+ : responsesReply("summary of the work", ASTRA_WIRE_USAGE);
+ }) as typeof globalThis.fetch;
+ try {
+ const text = await harness.models.compactHistory!({
+ session: { id: "compact-fallback" } as HarnessTurnInput["session"],
+ history: [
+ { seq: 1, kind: "user", payload: { text: "hello" }, createdAt: 1 },
+ { seq: 2, kind: "assistant", payload: { text: "hi" }, createdAt: 2 },
+ ] as unknown as SessionEntry[],
+ recordModelCall: () => {},
+ });
+ assert.match(text, /summary of the work/);
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+ assert.deepEqual(models, ["claude-sonnet-5", "gpt-6-sol"]);
+});
diff --git a/test/pi-harness-oneshot.test.ts b/test/pi-harness-oneshot.test.ts
index 7424e1b4c..1be785bb6 100644
--- a/test/pi-harness-oneshot.test.ts
+++ b/test/pi-harness-oneshot.test.ts
@@ -921,6 +921,14 @@ test("isProviderRefusal matches Anthropic's ToS-refusal wording and nothing else
true,
);
assert.equal(isProviderRefusal("This request seems to violate Anthropic’s usage policy."), true);
+ assert.equal(
+ isProviderRefusal(
+ "This request triggered restrictions on violative cyber content and was blocked under Anthropic's Usage Policy. To learn more, see https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback.",
+ ),
+ true,
+ );
+ assert.equal(isProviderRefusal("The model refused to complete the request"), true);
+ assert.equal(isProviderRefusal("Gateway model is unavailable: claude-opus-5-5"), true);
assert.equal(isProviderRefusal("Anthropic API error (overloaded_error): Overloaded"), false);
assert.equal(isProviderRefusal("prompt is too long: 250000 tokens > 200000 maximum"), false);
assert.equal(isProviderRefusal(undefined), false);
@@ -958,6 +966,12 @@ test("refusal fallback drawdown: Fable -> Opus, Opus -> Sonnet, never the refuse
assert.equal(refusalFallbackModelId("claude-fable-5-1"), "claude-opus-5");
assert.equal(refusalFallbackModelId("claude-fable-5"), "claude-opus-5");
assert.equal(refusalFallbackModelId("claude-opus-5"), "claude-sonnet-5");
+ assert.equal(refusalFallbackModelId("claude-opus-5", "gpt-6-sol"), "gpt-6-sol");
+ assert.equal(refusalFallbackModelId("gpt-6-sol", "gpt-6-sol"), "claude-opus-5");
+ assert.equal(
+ refusalFallbackModelId("claude-opus-5", "not-a-pi-model", (id) => id !== "not-a-pi-model"),
+ "claude-sonnet-5",
+ );
for (const id of REFUSAL_FALLBACK_MODEL_IDS) {
assert.notEqual(refusalFallbackModelId(id), id);
assert.equal(resolveModel(id)?.provider, "anthropic");
diff --git a/test/runtime-purpose.test.ts b/test/runtime-purpose.test.ts
index 4882ff570..a1d8b78bc 100644
--- a/test/runtime-purpose.test.ts
+++ b/test/runtime-purpose.test.ts
@@ -46,7 +46,7 @@ test("cron and loop entry points defer legacy injections until runtime resolutio
test("each purpose ignores conversation overrides and explicit fields including false win", async () => {
const config = setup();
const conversation = resolveRuntimeChoice(config, ORG, SCOPE, fallback);
- for (const purpose of ["cron", "subagent"] as const) {
+ for (const purpose of ["cron", "subagent", "fallback"] as const) {
await config.setPurposeRuntime(purpose, category);
assert.deepEqual(resolveRuntimeChoice(config, ORG, SCOPE, fallback, undefined, purpose), category);
assert.deepEqual(