Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): keep pre-round retry failures in attempt evidence
超超超超超级喜欢你的达妮娅8f9fe47139e1
diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.tsindex 799a9f118..88a5db8c1 100644--- a/workbench/l1/cassette.wb.ts+++ b/workbench/l1/cassette.wb.ts@@ -6,12 +6,14 @@// exactly, and a cassette that no longer matches the request says so instead of// answering a question the app no longer asks.-import { mkdtempSync, rmSync } from "node:fs";+import { existsSync, mkdtempSync, rmSync } from "node:fs";import { createServer } from "node:http";import { tmpdir } from "node:os";import { join } from "node:path";import { afterAll, describe, expect, it } from "vitest";import {+ adoptRetryEvidence,+ attemptsFromEndpoint,hashRequest,mergeRetryCassettes,readCassette,@@ -520,6 +522,83 @@ describe("record then replay", () => {{ attempt: 0, phase: "forward", status: "failed", detail: "timeout" },{ attempt: 1, phase: "complete", status: "recorded" },]);+ expect(+ adoptRetryEvidence(success, [+ [{ attempt: 0, phase: "forward", status: "failed", detail: "connection reset" }],+ success.attempts,+ ]).attempts,+ ).toEqual([+ { attempt: 0, phase: "forward", status: "failed", detail: "connection reset" },+ { attempt: 1, phase: "complete", status: "recorded" },+ ]);+ });++ it("keeps a pre-round forward failure when no attempt cassette exists", async () => {+ const failedFile = join(DIRECTORY, "retry-pre-round-0.json");+ const successFile = join(DIRECTORY, "retry-pre-round-1.json");+ const canonical = join(DIRECTORY, "retry-pre-round.json");+ const failedRecorder = await startRecorder({+ upstream: "http://127.0.0.1:59999",+ file: failedFile,+ scenario: "retry-pre-round",+ provider: "loopback",+ model: "loopback",+ });+ try {+ const response = await fetch(`${failedRecorder.url}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ model: "loopback",+ messages: [{ role: "user", content: "first" }],+ }),+ });+ expect(response.status).toBe(502);+ expect(existsSync(failedFile)).toBe(false);+ expect(existsSync(`${failedFile}.attempts.json`)).toBe(true);+ expect(failedRecorder.attempts).toEqual([+ expect.objectContaining({ phase: "forward", status: "failed" }),+ ]);+ } finally {+ failedRecorder.close();+ }++ const upstream = await startRawProvider({+ sse: 'data: {"model":"loopback"}\n\ndata: [DONE]\n\n',+ onRequest: () => undefined,+ });+ const successRecorder = await startRecorder({+ upstream: upstream.url,+ file: successFile,+ scenario: "retry-pre-round",+ provider: "loopback",+ model: "loopback",+ });+ try {+ const response = await fetch(`${successRecorder.url}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ model: "loopback",+ messages: [{ role: "user", content: "second" }],+ }),+ });+ expect(response.status).toBe(200);+ await response.text();+ } finally {+ successRecorder.close();+ upstream.close();+ }++ const success = readCassette(successFile);+ const adopted = adoptRetryEvidence(success, [+ attemptsFromEndpoint(failedRecorder),+ attemptsFromEndpoint(successRecorder),+ ]);+ writeCassette(canonical, adopted);+ expect(existsSync(failedFile)).toBe(false);+ expect(adopted.rounds).toEqual(success.rounds);+ expect(adopted.attempts?.map((attempt) => attempt.status)).toEqual(["failed", "recorded"]);});it("rejects a Chat cassette whose rounds resolve to different models", () => {diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.tsindex ae7cbd659..ccf12212c 100644--- a/workbench/lib/cassette.ts+++ b/workbench/lib/cassette.ts@@ -416,14 +416,34 @@ export function readCassette(file: string): Cassette {return cassette;}-export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {- const attempts = [- ...prior.flatMap((cassette) => cassette.attempts ?? []),- ...(success.attempts ?? []),- ].map((attempt, index) => ({ ...attempt, attempt: index }));+export function attemptsFromEndpoint(+ endpoint: Pick<ModelServerHandle, "attempts"> | undefined,+): CassetteAttempt[] {+ return (endpoint?.attempts ?? []).map((attempt) => ({+ attempt: attempt.attempt,+ phase: attempt.phase as CassetteAttempt["phase"],+ status: attempt.status as CassetteAttempt["status"],+ ...(typeof attempt.detail === "string" ? { detail: attempt.detail } : {}),+ }));+}++export function adoptRetryEvidence(+ success: Cassette,+ attemptLists: ReadonlyArray<readonly CassetteAttempt[] | undefined>,+): Cassette {+ const attempts = attemptLists+ .flatMap((list) => list ?? [])+ .map((attempt, index) => ({ ...attempt, attempt: index }));return { ...success, attempts };}+export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {+ return adoptRetryEvidence(success, [+ ...prior.map((cassette) => cassette.attempts),+ success.attempts,+ ]);+}+export function readCassetteEvidence(file: string): Cassette {try {return readCassette(file);diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.tsindex 4575131f3..e03c9853f 100644--- a/workbench/lib/runner.ts+++ b/workbench/lib/runner.ts@@ -17,7 +17,9 @@ import { getReasoningCapability } from "../../electron/ai-edition/deep-agent/chaimport type { LlmConfigStore } from "../../electron/ai-edition/llm-config-store";import type { AxcutDocument } from "../../src/lib/ai-edition/schema";import {- mergeRetryCassettes,+ adoptRetryEvidence,+ attemptsFromEndpoint,+ type CassetteAttempt,readCassetteEvidence,startRecorder,writeCassette,@@ -206,7 +208,8 @@ export async function runScenarioReps(for (let rep = 0; rep < options.reps; rep += 1) {budget?.refreshDeadline();const canonicalFile = options.live?.record?.(rep);- const priorCassettes: ReturnType<typeof readCassetteEvidence>[] = [];+ const attemptLists: CassetteAttempt[][] = [];+ let successCassette: ReturnType<typeof readCassetteEvidence> | undefined;let attempt = 0;for (;;) {const attemptFile =@@ -230,10 +233,11 @@ export async function runScenarioReps(maxRetries: env?.wireApi === "responses" ? 0 : undefined,});} finally {+ attemptLists.push(attemptsFromEndpoint(endpoint));endpoint?.close();}if (attemptFile && existsSync(attemptFile)) {- priorCassettes.push(readCassetteEvidence(attemptFile));+ successCassette = readCassetteEvidence(attemptFile);}const failureClass = result.scored.failureClass;if ((failureClass === "TIMEOUT" || failureClass === "TRANSPORT") && attempt < maxRetries) {@@ -241,9 +245,8 @@ export async function runScenarioReps(attempt += 1;continue;}- if (canonicalFile && priorCassettes.length > 0) {- const success = priorCassettes[priorCassettes.length - 1];- writeCassette(canonicalFile, mergeRetryCassettes(success, priorCassettes.slice(0, -1)));+ if (canonicalFile && successCassette) {+ writeCassette(canonicalFile, adoptRetryEvidence(successCassette, attemptLists));}results.push(result);options.onRepetition?.(result);