Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): keep pre-round retry failures in attempt evidence

超超超超超级喜欢你的达妮娅8f9fe47139e1
diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.ts
index 799a9f118..88a5db8c1 100644
--- a/workbench/l1/cassette.wb.ts
+++ b/workbench/l1/cassette.wb.ts
@@ -6,12 +6,14 @@
// exactly, and a cassette that no longer matches the request says so instead of
// answering a question the app no longer asks.
-import { mkdtempSync, rmSync } from "node:fs";
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import {
+ adoptRetryEvidence,
+ attemptsFromEndpoint,
hashRequest,
mergeRetryCassettes,
readCassette,
@@ -520,6 +522,83 @@ describe("record then replay", () => {
{ attempt: 0, phase: "forward", status: "failed", detail: "timeout" },
{ attempt: 1, phase: "complete", status: "recorded" },
]);
+ expect(
+ adoptRetryEvidence(success, [
+ [{ attempt: 0, phase: "forward", status: "failed", detail: "connection reset" }],
+ success.attempts,
+ ]).attempts,
+ ).toEqual([
+ { attempt: 0, phase: "forward", status: "failed", detail: "connection reset" },
+ { attempt: 1, phase: "complete", status: "recorded" },
+ ]);
+ });
+
+ it("keeps a pre-round forward failure when no attempt cassette exists", async () => {
+ const failedFile = join(DIRECTORY, "retry-pre-round-0.json");
+ const successFile = join(DIRECTORY, "retry-pre-round-1.json");
+ const canonical = join(DIRECTORY, "retry-pre-round.json");
+ const failedRecorder = await startRecorder({
+ upstream: "http://127.0.0.1:59999",
+ file: failedFile,
+ scenario: "retry-pre-round",
+ provider: "loopback",
+ model: "loopback",
+ });
+ try {
+ const response = await fetch(`${failedRecorder.url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ model: "loopback",
+ messages: [{ role: "user", content: "first" }],
+ }),
+ });
+ expect(response.status).toBe(502);
+ expect(existsSync(failedFile)).toBe(false);
+ expect(existsSync(`${failedFile}.attempts.json`)).toBe(true);
+ expect(failedRecorder.attempts).toEqual([
+ expect.objectContaining({ phase: "forward", status: "failed" }),
+ ]);
+ } finally {
+ failedRecorder.close();
+ }
+
+ const upstream = await startRawProvider({
+ sse: 'data: {"model":"loopback"}\n\ndata: [DONE]\n\n',
+ onRequest: () => undefined,
+ });
+ const successRecorder = await startRecorder({
+ upstream: upstream.url,
+ file: successFile,
+ scenario: "retry-pre-round",
+ provider: "loopback",
+ model: "loopback",
+ });
+ try {
+ const response = await fetch(`${successRecorder.url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ model: "loopback",
+ messages: [{ role: "user", content: "second" }],
+ }),
+ });
+ expect(response.status).toBe(200);
+ await response.text();
+ } finally {
+ successRecorder.close();
+ upstream.close();
+ }
+
+ const success = readCassette(successFile);
+ const adopted = adoptRetryEvidence(success, [
+ attemptsFromEndpoint(failedRecorder),
+ attemptsFromEndpoint(successRecorder),
+ ]);
+ writeCassette(canonical, adopted);
+ expect(existsSync(failedFile)).toBe(false);
+ expect(adopted.rounds).toEqual(success.rounds);
+ expect(adopted.attempts?.map((attempt) => attempt.status)).toEqual(["failed", "recorded"]);
});
it("rejects a Chat cassette whose rounds resolve to different models", () => {
diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.ts
index ae7cbd659..ccf12212c 100644
--- a/workbench/lib/cassette.ts
+++ b/workbench/lib/cassette.ts
@@ -416,14 +416,34 @@ export function readCassette(file: string): Cassette {
return cassette;
}
-export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {
- const attempts = [
- ...prior.flatMap((cassette) => cassette.attempts ?? []),
- ...(success.attempts ?? []),
- ].map((attempt, index) => ({ ...attempt, attempt: index }));
+export function attemptsFromEndpoint(
+ endpoint: Pick<ModelServerHandle, "attempts"> | undefined,
+): CassetteAttempt[] {
+ return (endpoint?.attempts ?? []).map((attempt) => ({
+ attempt: attempt.attempt,
+ phase: attempt.phase as CassetteAttempt["phase"],
+ status: attempt.status as CassetteAttempt["status"],
+ ...(typeof attempt.detail === "string" ? { detail: attempt.detail } : {}),
+ }));
+}
+
+export function adoptRetryEvidence(
+ success: Cassette,
+ attemptLists: ReadonlyArray<readonly CassetteAttempt[] | undefined>,
+): Cassette {
+ const attempts = attemptLists
+ .flatMap((list) => list ?? [])
+ .map((attempt, index) => ({ ...attempt, attempt: index }));
return { ...success, attempts };
}
+export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {
+ return adoptRetryEvidence(success, [
+ ...prior.map((cassette) => cassette.attempts),
+ success.attempts,
+ ]);
+}
+
export function readCassetteEvidence(file: string): Cassette {
try {
return readCassette(file);
diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.ts
index 4575131f3..e03c9853f 100644
--- a/workbench/lib/runner.ts
+++ b/workbench/lib/runner.ts
@@ -17,7 +17,9 @@ import { getReasoningCapability } from "../../electron/ai-edition/deep-agent/cha
import type { LlmConfigStore } from "../../electron/ai-edition/llm-config-store";
import type { AxcutDocument } from "../../src/lib/ai-edition/schema";
import {
- mergeRetryCassettes,
+ adoptRetryEvidence,
+ attemptsFromEndpoint,
+ type CassetteAttempt,
readCassetteEvidence,
startRecorder,
writeCassette,
@@ -206,7 +208,8 @@ export async function runScenarioReps(
for (let rep = 0; rep < options.reps; rep += 1) {
budget?.refreshDeadline();
const canonicalFile = options.live?.record?.(rep);
- const priorCassettes: ReturnType<typeof readCassetteEvidence>[] = [];
+ const attemptLists: CassetteAttempt[][] = [];
+ let successCassette: ReturnType<typeof readCassetteEvidence> | undefined;
let attempt = 0;
for (;;) {
const attemptFile =
@@ -230,10 +233,11 @@ export async function runScenarioReps(
maxRetries: env?.wireApi === "responses" ? 0 : undefined,
});
} finally {
+ attemptLists.push(attemptsFromEndpoint(endpoint));
endpoint?.close();
}
if (attemptFile && existsSync(attemptFile)) {
- priorCassettes.push(readCassetteEvidence(attemptFile));
+ successCassette = readCassetteEvidence(attemptFile);
}
const failureClass = result.scored.failureClass;
if ((failureClass === "TIMEOUT" || failureClass === "TRANSPORT") && attempt < maxRetries) {
@@ -241,9 +245,8 @@ export async function runScenarioReps(
attempt += 1;
continue;
}
- if (canonicalFile && priorCassettes.length > 0) {
- const success = priorCassettes[priorCassettes.length - 1];
- writeCassette(canonicalFile, mergeRetryCassettes(success, priorCassettes.slice(0, -1)));
+ if (canonicalFile && successCassette) {
+ writeCassette(canonicalFile, adoptRetryEvidence(successCassette, attemptLists));
}
results.push(result);
options.onRepetition?.(result);