Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): adopt only the retained attempt cassette

超超超超超级喜欢你的达妮娅17add81e63da
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index f9706d48f..13ee7c25a 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -564,19 +564,32 @@ describe("versioned measurement export and verification", () => {
);
});
- it("rejects a symlink or junction artifact when the platform can create one", ({ skip }) => {
+ it.skipIf(process.platform === "win32")("rejects a directory symlink artifact on posix", () => {
const base = createCandidate(root());
const outsideDirectory = join(dirname(base.runDir), "outside");
mkdirSync(outsideDirectory);
const outside = join(outsideDirectory, "outside.json");
writeFileSync(outside, "{}\n", "utf8");
const link = join(base.runDir, "linked");
- try {
- symlinkSync(outsideDirectory, link, process.platform === "win32" ? "junction" : "dir");
- } catch {
- skip();
- return;
- }
+ symlinkSync(outsideDirectory, link, "dir");
+ rewriteCandidate(base.runDir, (manifest) => {
+ manifest.artifacts[0] = {
+ role: "input-fixture",
+ path: "linked/outside.json",
+ sha256: sha256Bytes(readFileSync(outside)),
+ };
+ });
+ expectCode(() => exportValid(base), "SYMLINK_TRAVERSAL");
+ });
+
+ it.skipIf(process.platform !== "win32")("rejects a junction artifact on windows", () => {
+ const base = createCandidate(root());
+ const outsideDirectory = join(dirname(base.runDir), "outside");
+ mkdirSync(outsideDirectory);
+ const outside = join(outsideDirectory, "outside.json");
+ writeFileSync(outside, "{}\n", "utf8");
+ const link = join(base.runDir, "linked");
+ symlinkSync(outsideDirectory, link, "junction");
rewriteCandidate(base.runDir, (manifest) => {
manifest.artifacts[0] = {
role: "input-fixture",
diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.ts
index 88a5db8c1..0e1371d13 100644
--- a/workbench/l1/cassette.wb.ts
+++ b/workbench/l1/cassette.wb.ts
@@ -8,6 +8,7 @@
import { existsSync, mkdtempSync, rmSync } from "node:fs";
import { createServer } from "node:http";
+import { createServer as createNetServer } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
@@ -26,6 +27,7 @@ import { ENV_KEYS } from "../lib/env";
import { singleClip } from "../lib/fixtures";
import { normalizeIds, runScenario } from "../lib/harness";
import { startScriptedModel } from "../lib/model-server";
+import { adoptCassetteForRetainedAttempt } from "../lib/runner";
const DIRECTORY = mkdtempSync(join(tmpdir(), "wb-cassette-"));
const FILE = join(DIRECTORY, "wizard.json");
@@ -421,6 +423,47 @@ describe("record then replay", () => {
expect(() => replay.assertFresh()).not.toThrow();
});
+ it("replays a recorded Chat httpStatus instead of inventing 200", async () => {
+ const body = {
+ model: "legacy",
+ messages: [{ role: "user", content: "rate limited" }],
+ tools: [],
+ };
+ const file = join(DIRECTORY, "chat-http-status.json");
+ writeCassette(file, {
+ scenario: "chat-http-status",
+ provider: "loopback",
+ model: "legacy",
+ recordedAt: "2026-09-13T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: hashRequest(body),
+ digest: {
+ systemChars: 0,
+ toolCount: 0,
+ roles: ["user"],
+ lastUserText: "rate limited",
+ },
+ sse: "data: [DONE]\n\n",
+ httpStatus: 429,
+ },
+ ],
+ });
+ const replay = await startReplay({ file });
+ try {
+ const response = await fetch(`${replay.url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify(body),
+ });
+ expect(response.status).toBe(429);
+ expect(replay.servedRounds[0]?.status).toBe(429);
+ } finally {
+ replay.close();
+ }
+ });
+
it("leaves usage unknown when no valid object appears", () => {
const sse = [
`data: ${JSON.stringify({ usage: null })}\n\n`,
@@ -537,8 +580,20 @@ describe("record then replay", () => {
const failedFile = join(DIRECTORY, "retry-pre-round-0.json");
const successFile = join(DIRECTORY, "retry-pre-round-1.json");
const canonical = join(DIRECTORY, "retry-pre-round.json");
+ const occupied = createNetServer((socket) => {
+ socket.destroy();
+ });
+ await new Promise<void>((resolve, reject) => {
+ occupied.once("error", reject);
+ occupied.listen(0, "127.0.0.1", () => resolve());
+ });
+ const occupiedAddress = occupied.address();
+ if (!occupiedAddress || typeof occupiedAddress === "string") {
+ occupied.close();
+ throw new Error("occupied upstream has no address");
+ }
const failedRecorder = await startRecorder({
- upstream: "http://127.0.0.1:59999",
+ upstream: `http://127.0.0.1:${occupiedAddress.port}`,
file: failedFile,
scenario: "retry-pre-round",
provider: "loopback",
@@ -561,6 +616,7 @@ describe("record then replay", () => {
]);
} finally {
failedRecorder.close();
+ await new Promise<void>((resolve) => occupied.close(() => resolve()));
}
const upstream = await startRawProvider({
@@ -601,6 +657,40 @@ describe("record then replay", () => {
expect(adopted.attempts?.map((attempt) => attempt.status)).toEqual(["failed", "recorded"]);
});
+ it("adopts only the retained attempt tape, never an earlier retry cassette", () => {
+ const earlier = join(DIRECTORY, "retained-attempt-0.json");
+ writeCassette(earlier, {
+ scenario: "retained-attempt",
+ provider: "loopback",
+ model: "loopback",
+ recordedAt: "2026-09-13T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: "0123456789abcdef",
+ digest: {
+ systemChars: 0,
+ toolCount: 0,
+ roles: ["user"],
+ lastUserText: "attempt-0",
+ },
+ sse: "data: [DONE]\n\n",
+ },
+ ],
+ });
+ const lists = [
+ [{ attempt: 0, phase: "forward" as const, status: "failed" as const, detail: "timeout" }],
+ [],
+ ];
+ expect(adoptCassetteForRetainedAttempt(undefined, lists)).toBeUndefined();
+ expect(
+ adoptCassetteForRetainedAttempt(join(DIRECTORY, "retained-attempt-1.json"), lists),
+ ).toBeUndefined();
+ expect(adoptCassetteForRetainedAttempt(earlier, lists)?.rounds[0]?.digest.lastUserText).toBe(
+ "attempt-0",
+ );
+ });
+
it("rejects a Chat cassette whose rounds resolve to different models", () => {
expect(() =>
writeCassette(join(DIRECTORY, "mixed-models.json"), {
diff --git a/workbench/l1/responses-cli.wb.ts b/workbench/l1/responses-cli.wb.ts
index c23fb1f7f..9821125ad 100644
--- a/workbench/l1/responses-cli.wb.ts
+++ b/workbench/l1/responses-cli.wb.ts
@@ -1,5 +1,5 @@
import { spawnSync } from "node:child_process";
-import { readFileSync } from "node:fs";
+import { readFileSync, rmSync } from "node:fs";
import { buildSync } from "esbuild";
import { describe, expect, it } from "vitest";
import { readCassette, startRecorder } from "../lib/cassette";
@@ -72,6 +72,7 @@ describe("offline main CLI replay", () => {
const cleanEnv = Object.fromEntries(
Object.entries(process.env).filter(([name]) => !envNames.has(name)),
);
+ rmSync(`workbench/runs/${LABEL}`, { recursive: true, force: true });
const child = spawnSync(
process.execPath,
[
diff --git a/workbench/l1/responses-record-replay.wb.ts b/workbench/l1/responses-record-replay.wb.ts
index 70494b49d..8e0f5c0ab 100644
--- a/workbench/l1/responses-record-replay.wb.ts
+++ b/workbench/l1/responses-record-replay.wb.ts
@@ -130,8 +130,46 @@ describe("native Responses record and replay", () => {
});
it("rejects omitted, unknown, and Chat-disguised Responses discriminators", () => {
- const source = JSON.parse(readFileSync(`${ROOT}/native-tool-loop.json`, "utf8"));
- const writeTamper = (name: string, mutate: (value: typeof source) => void): string => {
+ type TamperCassette = {
+ wireApi?: string;
+ transport?: unknown;
+ rounds: Array<Record<string, unknown>>;
+ [key: string]: unknown;
+ };
+ const source = {
+ scenario: "discriminator-base",
+ provider: "loopback",
+ model: "gpt-5-test",
+ recordedAt: "2026-09-12T00:00:00.000Z",
+ wireApi: "responses",
+ transport: transportIdentity({ wireApi: "responses", maxOutputTokens: 2048 }),
+ rounds: [
+ {
+ round: 0,
+ requestHash: "discriminator-hash",
+ digest: {
+ systemChars: 0,
+ toolCount: 0,
+ roles: ["user"],
+ lastUserText: "Explain Responses",
+ },
+ sse: 'data: {"type":"response.completed"}\n\n',
+ inboundPath: "/v1/responses",
+ requestBodyBase64: Buffer.from("{}").toString("base64"),
+ requestByteCount: 2,
+ requestSha256: "0".repeat(64),
+ response: {
+ bodyBase64: Buffer.from('data: {"type":"response.completed"}\n\n').toString("base64"),
+ byteCount: 36,
+ sha256: "0".repeat(64),
+ status: 200,
+ contentType: "text/event-stream",
+ },
+ terminalStatus: "completed",
+ },
+ ],
+ } as TamperCassette;
+ const writeTamper = (name: string, mutate: (value: TamperCassette) => void): string => {
const value = structuredClone(source);
mutate(value);
const file = `${ROOT}/discriminator-${name}-${randomUUID()}.json`;
@@ -258,7 +296,41 @@ describe("native Responses record and replay", () => {
});
it("rejects byte-attestation drift and sends one request on provider failure", async () => {
- const file = `${ROOT}/non-2xx-bytes.json`;
+ const file = `${ROOT}/isolated-non-2xx-bytes.json`;
+ const responseText = '{"error":"split 🙂 body"}';
+ const bytes = Buffer.from(responseText);
+ const upstreamForCassette = createServer((_req, res) => {
+ res.writeHead(429, { "content-type": "application/problem+json; charset=utf-8" });
+ res.end(bytes);
+ });
+ await new Promise<void>((resolve) => upstreamForCassette.listen(0, "127.0.0.1", resolve));
+ const cassetteAddress = upstreamForCassette.address();
+ if (!cassetteAddress || typeof cassetteAddress === "string") throw new Error("no address");
+ const cassetteProfile = profile(1);
+ const cassetteRecorder = await startRecorder({
+ upstream: `http://127.0.0.1:${cassetteAddress.port}`,
+ file,
+ scenario: "byte-failure",
+ provider: "loopback",
+ model: "gpt-5-test",
+ wireApi: "responses",
+ transport: cassetteProfile.transport,
+ publicHeaders: cassetteProfile.headers,
+ });
+ try {
+ await fetch(`${cassetteRecorder.url}/responses`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ model: "gpt-5-test",
+ input: [],
+ reasoning: { effort: "medium" },
+ }),
+ });
+ } finally {
+ cassetteRecorder.close();
+ upstreamForCassette.close();
+ }
const tampered = `${ROOT}/tampered.json`;
const value = JSON.parse(readFileSync(file, "utf8"));
value.rounds[0].response.sha256 = "0".repeat(64);
diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.ts
index ccf12212c..3cfd01150 100644
--- a/workbench/lib/cassette.ts
+++ b/workbench/lib/cassette.ts
@@ -914,12 +914,13 @@ export async function startReplay(options: {
res.end(bytes);
return;
}
- res.writeHead(200, { "content-type": "text/event-stream" });
+ const httpStatus = stored.httpStatus ?? 200;
+ res.writeHead(httpStatus, { "content-type": "text/event-stream" });
servedRounds.push({
round: myRound,
terminalStatus: "legacy-or-unknown",
usage: usageFromSse(stored.sse) ?? null,
- status: 200,
+ status: httpStatus,
contentType: "text/event-stream",
bodySha256: sha256(Buffer.from(stored.sse)),
});
diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.ts
index e03c9853f..ac074a742 100644
--- a/workbench/lib/runner.ts
+++ b/workbench/lib/runner.ts
@@ -19,6 +19,7 @@ import type { AxcutDocument } from "../../src/lib/ai-edition/schema";
import {
adoptRetryEvidence,
attemptsFromEndpoint,
+ type Cassette,
type CassetteAttempt,
readCassetteEvidence,
startRecorder,
@@ -173,6 +174,15 @@ export interface ScenarioRepsOptions {
onRepetition?: (result: RepetitionResult) => void;
}
+/** Only the retained attempt's tape may become canonical. */
+export function adoptCassetteForRetainedAttempt(
+ retainedAttemptFile: string | undefined,
+ attemptLists: CassetteAttempt[][],
+): Cassette | undefined {
+ if (!retainedAttemptFile || !existsSync(retainedAttemptFile)) return undefined;
+ return adoptRetryEvidence(readCassetteEvidence(retainedAttemptFile), attemptLists);
+}
+
/**
* Runs a scenario `reps` times, sequentially.
*
@@ -209,7 +219,6 @@ export async function runScenarioReps(
budget?.refreshDeadline();
const canonicalFile = options.live?.record?.(rep);
const attemptLists: CassetteAttempt[][] = [];
- let successCassette: ReturnType<typeof readCassetteEvidence> | undefined;
let attempt = 0;
for (;;) {
const attemptFile =
@@ -236,17 +245,15 @@ export async function runScenarioReps(
attemptLists.push(attemptsFromEndpoint(endpoint));
endpoint?.close();
}
- if (attemptFile && existsSync(attemptFile)) {
- successCassette = readCassetteEvidence(attemptFile);
- }
const failureClass = result.scored.failureClass;
if ((failureClass === "TIMEOUT" || failureClass === "TRANSPORT") && attempt < maxRetries) {
discarded.push(result);
attempt += 1;
continue;
}
- if (canonicalFile && successCassette) {
- writeCassette(canonicalFile, adoptRetryEvidence(successCassette, attemptLists));
+ if (canonicalFile) {
+ const adopted = adoptCassetteForRetainedAttempt(attemptFile, attemptLists);
+ if (adopted) writeCassette(canonicalFile, adopted);
}
results.push(result);
options.onRepetition?.(result);