Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Stop goal rounds after a model call fails at the provider (#2041)

Joshua France01433d5b2ddc
diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts
index d0822f425..02f5c3eb4 100644
--- a/src/harness/pi-harness.ts
+++ b/src/harness/pi-harness.ts
@@ -1009,12 +1009,18 @@ export function piLastAssistantTextOrThrow(session: AssistantTextSession): strin
return session.getLastAssistantText();
}
+function messagesSince(session: AssistantTextSession, messagesBefore?: number): AssistantTextSession {
+ return messagesBefore === undefined
+ ? session
+ : ({ messages: session.messages.slice(messagesBefore) } as AssistantTextSession);
+}
+
+function piRoundFailed(session: AssistantTextSession, messagesBefore: number): boolean {
+ return !!piFailedAssistant(messagesSince(session, messagesBefore));
+}
+
export function piTurnError(session: AssistantTextSession, thrown: unknown, messagesBefore?: number): Error {
- const fresh =
- messagesBefore === undefined
- ? session
- : ({ messages: session.messages.slice(messagesBefore) } as AssistantTextSession);
- const detailed = piAssistantFailure(fresh);
+ const detailed = piAssistantFailure(messagesSince(session, messagesBefore));
if (detailed) return detailed;
return thrown instanceof Error ? thrown : new Error(String(thrown));
}
@@ -1027,11 +1033,7 @@ export function isProviderRefusal(message: string | undefined): boolean {
}
export function providerRefusalError(session: AssistantTextSession, messagesBefore?: number): string | null {
- const fresh =
- messagesBefore === undefined
- ? session
- : ({ messages: session.messages.slice(messagesBefore) } as AssistantTextSession);
- const err = piAssistantError(fresh);
+ const err = piAssistantError(messagesSince(session, messagesBefore));
return err && isProviderRefusal(err) ? err : null;
}
@@ -2250,15 +2252,17 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
const rawRemainingCapMs = floorCap.remainingCapMs;
const raceCapMs = floorCap.raceCapMs;
const extendCapMs = floorCap.extendMs;
+ const refusedModelIds = new Set<string>();
const attemptRefusalFallback = async (refusal: string): Promise<boolean> => {
if (userAborted || turn.cancel?.aborted) return false;
const fromId = (entry.agentSession.model as { id?: string } | undefined)?.id;
+ if (fromId) refusedModelIds.add(fromId);
const configured = opts?.resolveFallbackRuntime?.();
const fallbackId = fromId
? refusalFallbackModelId(fromId, configured?.modelId, (id) => !!resolveModel(id, !turn.providerKeys))
: undefined;
const fallback = fallbackId ? resolveModel(fallbackId, !turn.providerKeys) : undefined;
- if (!fallbackId || !fallback) return false;
+ if (!fallbackId || !fallback || refusedModelIds.has(fallbackId)) return false;
const capMs = raceCapMs();
if (turnWallClockMs > 0 && capMs < EMPTY_ENDING_MIN_BUDGET_MS) return false;
console.error(
@@ -2291,6 +2295,17 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
if (outcome !== "ok") throw new NonRetryableTurnError(refusal);
return true;
};
+ const recoverRefusedRound = async (): Promise<void> => {
+ if (entry.ref.runtimeHandoff || userAborted || turn.cancel?.aborted) return;
+ const refusal = providerRefusalError(entry.agentSession, messagesBefore);
+ if (!refusal) return;
+ try {
+ await attemptRefusalFallback(refusal);
+ } catch (e) {
+ swallow("pi: refusal fallback", e);
+ throw new NonRetryableTurnError(refusal);
+ }
+ };
try {
const images = turn.images?.length
? turn.images.map((i) => ({ type: "image" as const, data: i.dataBase64, mimeType: i.mimeType }))
@@ -2305,6 +2320,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
abort: () => entry.agentSession.abort(),
},
);
+ if (wallClock === "ok") await recoverRefusedRound();
const goalAfterPrompt = entry.ref.goal;
if (
wallClock === "ok" &&
@@ -2323,7 +2339,8 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
!!turn.cancel?.aborted ||
!!entry.ref.runtimeHandoff ||
!!entry.ref.pausedOnApproval ||
- !!entry.ref.pendingApprovals?.length,
+ !!entry.ref.pendingApprovals?.length ||
+ piRoundFailed(entry.agentSession, messagesBefore),
beforePrompt: async (note) => {
console.error(
`[goal] continuation session=${turn.session.id} round=${(entry.ref.goalRound ?? 0) + 1}`,
@@ -2333,28 +2350,18 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness {
entry.ref.silentRequested = false;
await thinkTail;
},
- prompt: (note) => {
- if (turnWallClockMs > 0 && rawRemainingCapMs() < EMPTY_ENDING_MIN_BUDGET_MS)
- return Promise.resolve<TurnWallClockOutcome>("aborted");
- return raceTurnWallClock(entry.agentSession.prompt(note), {
+ prompt: async (note) => {
+ if (turnWallClockMs > 0 && rawRemainingCapMs() < EMPTY_ENDING_MIN_BUDGET_MS) return "aborted";
+ const outcome = await raceTurnWallClock(entry.agentSession.prompt(note), {
capMs: raceCapMs(),
extendMs: extendCapMs,
abort: () => entry.agentSession.abort(),
});
+ if (outcome === "ok") await recoverRefusedRound();
+ return outcome;
},
});
}
- if (wallClock === "ok" && !entry.ref.runtimeHandoff && !userAborted && !turn.cancel?.aborted) {
- const refusal = providerRefusalError(entry.agentSession, messagesBefore);
- if (refusal) {
- try {
- await attemptRefusalFallback(refusal);
- } catch (e) {
- swallow("pi: refusal fallback", e);
- throw new NonRetryableTurnError(refusal);
- }
- }
- }
const note = emptyEndingNote({
wallClock,
userAborted,
diff --git a/test/pi-harness-goal-provider-failure.test.ts b/test/pi-harness-goal-provider-failure.test.ts
new file mode 100644
index 000000000..ceadad8f5
--- /dev/null
+++ b/test/pi-harness-goal-provider-failure.test.ts
@@ -0,0 +1,182 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createGoalRecord, latestGoalRecord } from "../src/harness/goal.ts";
+import { createPiHarness } from "../src/harness/pi-harness.ts";
+import { NonRetryableTurnError } from "../src/core/turn-error.ts";
+import type { HarnessTurnInput } from "../src/harness/harness.ts";
+import type { NewEntry } from "../src/sessions/session-store.ts";
+import type { SessionEntry } from "../src/types.ts";
+
+function goalTurn(
+ sessionId: string,
+ cancel: AbortSignal,
+ entries: Array<{ type: string; payload: unknown }>,
+): HarnessTurnInput {
+ let seq = 0;
+ const goal = createGoalRecord({ objective: "keep researching until done" });
+ return {
+ session: { id: sessionId } as HarnessTurnInput["session"],
+ cancel,
+ input: "continue",
+ systemPrompt: "BASE",
+ history: [
+ {
+ type: "tool_result",
+ payload: { tool: "goal", action: "create", goal },
+ seq: 0,
+ sessionId,
+ parentSeq: null,
+ createdAt: 1,
+ scopeLabel: "personal:tester" as HarnessTurnInput["scopeLabel"],
+ },
+ ],
+ tools: {} as HarnessTurnInput["tools"],
+ scopeLabel: "personal:tester" as HarnessTurnInput["scopeLabel"],
+ orgScopeId: "org:test" as HarnessTurnInput["orgScopeId"],
+ emit: async (entry: NewEntry) => {
+ entries.push({ type: entry.type, payload: entry.payload });
+ return { ...entry, seq: ++seq, createdAt: Date.now() } as unknown as SessionEntry;
+ },
+ recordModelCall: () => {},
+ };
+}
+
+test("an active goal stops continuing once a model round fails at the provider", async () => {
+ const harness = createPiHarness({ apiKey: "sk-test" });
+ const controller = new AbortController();
+ const entries: Array<{ type: string; payload: unknown }> = [];
+ const realFetch = globalThis.fetch;
+ let requests = 0;
+ globalThis.fetch = (async () => {
+ requests++;
+ if (requests > 5) controller.abort();
+ return new Response(
+ JSON.stringify({
+ type: "error",
+ error: { type: "invalid_request_error", message: "Your credit balance is too low." },
+ }),
+ { status: 400, headers: { "content-type": "application/json" } },
+ );
+ }) as typeof globalThis.fetch;
+ try {
+ const outcome = await harness.turns.runTurn(goalTurn("goal-provider-failure", controller.signal, entries)).then(
+ (result) => ({ result }),
+ (error: unknown) => ({ error }),
+ );
+ assert.equal(requests, 1, "a failed round must not be re-prompted by the goal loop");
+ assert.ok(
+ "error" in outcome && outcome.error instanceof NonRetryableTurnError,
+ "the turn fails with the provider error",
+ );
+ assert.match((outcome as { error: Error }).error.message, /credit balance is too low/);
+ assert.equal(latestGoalRecord(entries)?.status, "active", "the goal itself stays active for a later turn");
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+});
+
+function textStream(text: string): Response {
+ const event = (name: string, data: unknown) => `event: ${name}\ndata: ${JSON.stringify(data)}\n\n`;
+ return new Response(
+ event("message_start", {
+ type: "message_start",
+ message: {
+ id: "m",
+ type: "message",
+ role: "assistant",
+ content: [],
+ model: "x",
+ stop_reason: null,
+ usage: { input_tokens: 5, output_tokens: 0 },
+ },
+ }) +
+ event("content_block_start", {
+ type: "content_block_start",
+ index: 0,
+ content_block: { type: "text", text: "" },
+ }) +
+ event("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text } }) +
+ event("content_block_stop", { type: "content_block_stop", index: 0 }) +
+ event("message_delta", {
+ type: "message_delta",
+ delta: { stop_reason: "end_turn" },
+ usage: { output_tokens: 3 },
+ }) +
+ event("message_stop", { type: "message_stop" }),
+ { status: 200, headers: { "content-type": "text/event-stream" } },
+ );
+}
+
+test("a refused goal round falls back to another model and the goal keeps going", async () => {
+ const harness = createPiHarness({ apiKey: "sk-test" });
+ const controller = new AbortController();
+ const entries: Array<{ type: string; payload: unknown }> = [];
+ const realFetch = globalThis.fetch;
+ const models: string[] = [];
+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+ const model = (JSON.parse(String(init?.body ?? "{}")) as { model: string }).model;
+ models.push(model);
+ if (models.length >= 5) controller.abort();
+ if (models.length >= 2 && model === models[0])
+ return new Response(
+ JSON.stringify({
+ type: "error",
+ error: {
+ type: "invalid_request_error",
+ message: "API integrators: you can reduce refusals for your users by configuring a fallback model",
+ },
+ }),
+ { status: 400, headers: { "content-type": "application/json" } },
+ );
+ return textStream(`reply ${models.length}`);
+ }) as typeof globalThis.fetch;
+ try {
+ await harness.turns.runTurn(goalTurn("goal-refusal-fallback", controller.signal, entries)).catch(() => undefined);
+ const [primary, refused, fallback, continued] = models;
+ assert.equal(refused, primary, "the second round goes to the primary model and is refused");
+ assert.notEqual(fallback, primary, "the refused round is answered by a fallback model");
+ assert.equal(continued, fallback, "the goal keeps going on the fallback model instead of stopping");
+ assert.equal(latestGoalRecord(entries)?.status, "active");
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+});
+
+test("a goal never falls back to a model that already refused in the same turn", async () => {
+ const harness = createPiHarness({ apiKey: "sk-test" });
+ const controller = new AbortController();
+ const entries: Array<{ type: string; payload: unknown }> = [];
+ const realFetch = globalThis.fetch;
+ const models: string[] = [];
+ globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
+ const request = JSON.parse(String(init?.body ?? "{}")) as { model: string; messages: unknown[] };
+ models.push(request.model);
+ if (models.length >= 8) controller.abort();
+ const latest = JSON.stringify(request.messages.at(-1));
+ if (models.length === 1 || latest.includes("could not answer this request"))
+ return textStream(`reply ${models.length}`);
+ return new Response(
+ JSON.stringify({
+ type: "error",
+ error: {
+ type: "invalid_request_error",
+ message: "API integrators: you can reduce refusals for your users by configuring a fallback model",
+ },
+ }),
+ { status: 400, headers: { "content-type": "application/json" } },
+ );
+ }) as typeof globalThis.fetch;
+ try {
+ const outcome = await harness.turns.runTurn(goalTurn("goal-refusal-alternation", controller.signal, entries)).then(
+ (result) => ({ result }),
+ (error: unknown) => ({ error }),
+ );
+ const [primary, refused, fallback, refusedAgain] = models;
+ assert.deepEqual([refused, refusedAgain], [primary, fallback]);
+ assert.equal(models.length, 4, "the turn ends instead of bouncing back to the model that already refused");
+ assert.ok("error" in outcome && outcome.error instanceof NonRetryableTurnError);
+ assert.equal(latestGoalRecord(entries)?.status, "active");
+ } finally {
+ globalThis.fetch = realFetch;
+ }
+});