Skip to content

Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.

fix(workbench): bind retry evidence, fingerprints, and Chat model identity

超超超超超级喜欢你的达妮娅70ec779b97a8
diff --git a/workbench/cli.ts b/workbench/cli.ts
index 04fa926c9..b862690ed 100644
--- a/workbench/cli.ts
+++ b/workbench/cli.ts
@@ -569,7 +569,6 @@ async function commandJudge(options: Options): Promise<number> {
if (!liveJudgeTransport || !liveJudgeBudget) {
throw new Error("live judge transport was not prepared");
}
- liveJudgeBudget.refreshDeadline();
endpoint = await startRecorder({
// Le proxy, pas le provider en direct — même règle que `runner.ts` :
// c'est le seul endroit d'où une cassette peut sortir, et le seul
@@ -612,6 +611,7 @@ async function commandJudge(options: Options): Promise<number> {
const context = contextFromPersistedTurn(turn);
const readings = new Map<string, JudgeReading>();
for (const judged of scenario.judged ?? []) {
+ liveJudgeBudget?.refreshDeadline();
const reading = await askJudge({
endpoint: {
baseUrl: endpoint.url,
diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.ts
index 93a9c3059..f9706d48f 100644
--- a/workbench/l0/measurements.wb.ts
+++ b/workbench/l0/measurements.wb.ts
@@ -15,6 +15,7 @@ import {
type ArtifactReference,
assertComparable,
boundBaselineFromMeasurement,
+ boundMeasurementFingerprints,
exportMeasurement,
MeasurementError,
type MeasurementManifest,
@@ -33,6 +34,7 @@ import {
} from "../lib/provenance";
import { wilson95 } from "../lib/stats";
import { runMeasurementCli } from "../measurement-cli";
+import { getScenario } from "../scenarios/registry";
const roots: string[] = [];
const cliOutputs: string[] = [];
@@ -186,11 +188,13 @@ function createCandidate(root: string): {
telemetrySha256: fixture.telemetrySha256,
},
fingerprints: {
- promptSha256: HASH("1"),
+ ...boundMeasurementFingerprints(getScenario("target-right-clip"), {
+ systemSha256: HASH("d"),
+ toolsSha256: HASH("e"),
+ toolNames: [],
+ }),
systemSha256: HASH("d"),
toolsSha256: HASH("e"),
- wireSha256: HASH("4"),
- rubricSha256: HASH("5"),
},
models: {
agent: { requested: "loopback-model", observed: "loopback-model" },
@@ -661,9 +665,30 @@ describe("measurement identity and bound baseline", () => {
const changedSource = structuredClone(baseline);
changedSource.effectiveSourceSha256 = HASH("0");
expectCode(() => verifyBoundBaseline(changedSource, verified), "SOURCE_IDENTITY_MISMATCH");
+ const changedFailures = structuredClone(baseline);
+ changedFailures.expectedFailures = ["whatever-I-want"];
+ expectCode(() => verifyBoundBaseline(changedFailures, verified), "CHECK_COUNT_MISMATCH");
expect(() => assertComparable(baseline.identity, verified.manifest)).not.toThrow();
});
+ it("rejects a rewritten prompt, wire, or rubric fingerprint", () => {
+ const fingerprints = createCandidate(root());
+ rewriteCandidate(fingerprints.runDir, (manifest) => {
+ manifest.fingerprints.promptSha256 = HASH("1");
+ });
+ expectCode(() => exportValid(fingerprints), "INCOMPATIBLE_IDENTITIES");
+ const wire = createCandidate(root());
+ rewriteCandidate(wire.runDir, (manifest) => {
+ manifest.fingerprints.wireSha256 = HASH("4");
+ });
+ expectCode(() => exportValid(wire), "INCOMPATIBLE_IDENTITIES");
+ const rubric = createCandidate(root());
+ rewriteCandidate(rubric.runDir, (manifest) => {
+ manifest.fingerprints.rubricSha256 = HASH("5");
+ });
+ expectCode(() => exportValid(rubric), "INCOMPATIBLE_IDENTITIES");
+ });
+
it("never turns an explicit rejected review into baseline approval", () => {
const base = createCandidate(root());
const receipt = JSON.parse(readFileSync(base.review, "utf8")) as { disposition: string };
diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.ts
index 73f5aac70..799a9f118 100644
--- a/workbench/l1/cassette.wb.ts
+++ b/workbench/l1/cassette.wb.ts
@@ -13,6 +13,7 @@ import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import {
hashRequest,
+ mergeRetryCassettes,
readCassette,
startRecorder,
startReplay,
@@ -489,4 +490,110 @@ describe("record then replay", () => {
target.close();
}
});
+
+ it("keeps failed retry attempts on the successful cassette", () => {
+ const digest = { systemChars: 1, toolCount: 0, roles: ["user"], lastUserText: "retry" };
+ const failed = {
+ scenario: "retry-probe",
+ provider: "loopback",
+ model: "loopback",
+ recordedAt: "2026-09-13T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: "0123456789abcdef",
+ digest,
+ sse: "data: [DONE]\n\n",
+ },
+ ],
+ attempts: [
+ { attempt: 0, phase: "forward" as const, status: "failed" as const, detail: "timeout" },
+ ],
+ };
+ const success = {
+ ...failed,
+ attempts: [{ attempt: 0, phase: "complete" as const, status: "recorded" as const }],
+ };
+ const merged = mergeRetryCassettes(success, [failed]);
+ expect(merged.rounds).toEqual(success.rounds);
+ expect(merged.attempts).toEqual([
+ { attempt: 0, phase: "forward", status: "failed", detail: "timeout" },
+ { attempt: 1, phase: "complete", status: "recorded" },
+ ]);
+ });
+
+ it("rejects a Chat cassette whose rounds resolve to different models", () => {
+ expect(() =>
+ writeCassette(join(DIRECTORY, "mixed-models.json"), {
+ scenario: "mixed-models",
+ provider: "loopback",
+ model: "alias",
+ resolvedModel: "gpt-a",
+ recordedAt: "2026-09-13T00:00:00.000Z",
+ rounds: [
+ {
+ round: 0,
+ requestHash: "0123456789abcdef",
+ digest: { systemChars: 0, toolCount: 0, roles: ["user"], lastUserText: "a" },
+ sse: `data: ${JSON.stringify({ model: "gpt-a" })}\n\ndata: [DONE]\n\n`,
+ },
+ {
+ round: 1,
+ requestHash: "fedcba9876543210",
+ digest: { systemChars: 0, toolCount: 0, roles: ["user"], lastUserText: "b" },
+ sse: `data: ${JSON.stringify({ model: "gpt-b" })}\n\ndata: [DONE]\n\n`,
+ },
+ ],
+ }),
+ ).toThrow(/model mismatch/);
+ });
+
+ it("fails closed when the concrete Chat model changes mid-measurement", async () => {
+ let round = 0;
+ const upstream = createServer((_req, res) => {
+ const model = round === 0 ? "gpt-a" : "gpt-b";
+ round += 1;
+ res.writeHead(200, { "content-type": "text/event-stream" });
+ res.end(`data: ${JSON.stringify({ model })}\n\ndata: [DONE]\n\n`);
+ });
+ await new Promise<void>((resolve) => upstream.listen(0, "127.0.0.1", resolve));
+ const address = upstream.address();
+ if (!address || typeof address === "string")
+ throw new Error("model-change upstream has no address");
+ const file = join(DIRECTORY, "model-change.json");
+ const recorder = await startRecorder({
+ upstream: `http://127.0.0.1:${address.port}`,
+ file,
+ scenario: "model-change",
+ provider: "loopback",
+ model: "alias",
+ });
+ try {
+ const first = await fetch(`${recorder.url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ model: "alias",
+ messages: [{ role: "user", content: "first" }],
+ }),
+ });
+ expect(first.status).toBe(200);
+ await first.text();
+ const second = await fetch(`${recorder.url}/chat/completions`, {
+ method: "POST",
+ headers: { "content-type": "application/json" },
+ body: JSON.stringify({
+ model: "alias",
+ messages: [{ role: "user", content: "second" }],
+ }),
+ });
+ expect(second.status).toBe(502);
+ expect(await second.json()).toEqual({
+ error: "workbench proxy: resolved model changed mid-measurement",
+ });
+ } finally {
+ recorder.close();
+ upstream.close();
+ }
+ });
});
diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.ts
index 3b574f8e2..ae7cbd659 100644
--- a/workbench/lib/cassette.ts
+++ b/workbench/lib/cassette.ts
@@ -345,6 +345,7 @@ export function validateCassette(cassette: Cassette): void {
"cassette has transport or Responses evidence without a wireApi discriminator",
);
}
+ assertChatResolvedModel(cassette);
return;
}
if (declaredWireApi !== "chat-completions" && declaredWireApi !== "responses") {
@@ -369,6 +370,7 @@ export function validateCassette(cassette: Cassette): void {
if (hasResponsesEvidence || hasResponsesSseEvents) {
throw new Error("Chat cassette contains Responses-only round evidence");
}
+ assertChatResolvedModel(cassette);
return;
}
for (const round of cassette.rounds) {
@@ -399,12 +401,37 @@ export function validateCassette(cassette: Cassette): void {
}
}
+function assertChatResolvedModel(cassette: Cassette): void {
+ for (const round of cassette.rounds) {
+ const model = modelFromSse(round.sse, "chat-completions");
+ if (model && cassette.resolvedModel && model !== cassette.resolvedModel) {
+ throw new Error(`Chat round ${round.round} model mismatch`);
+ }
+ }
+}
+
export function readCassette(file: string): Cassette {
const cassette = JSON.parse(readFileSync(file, "utf8")) as Cassette;
validateCassette(cassette);
return cassette;
}
+export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {
+ const attempts = [
+ ...prior.flatMap((cassette) => cassette.attempts ?? []),
+ ...(success.attempts ?? []),
+ ].map((attempt, index) => ({ ...attempt, attempt: index }));
+ return { ...success, attempts };
+}
+
+export function readCassetteEvidence(file: string): Cassette {
+ try {
+ return readCassette(file);
+ } catch {
+ return JSON.parse(readFileSync(file, "utf8")) as Cassette;
+ }
+}
+
export function writeCassette(file: string, cassette: Cassette): void {
validateCassette(cassette);
const payload = `${JSON.stringify(cassette, null, "\t")}\n`;
@@ -695,7 +722,21 @@ export async function startRecorder(options: {
res.end('{"error":"workbench proxy: invalid Responses terminal sequence"}');
return;
}
- resolvedModel ??= terminal?.model ?? modelFromSse(decoded, wireApi);
+ const observedModel = terminal?.model ?? modelFromSse(decoded, wireApi);
+ if (observedModel) {
+ if (resolvedModel && observedModel !== resolvedModel) {
+ retainAttempt({
+ attempt,
+ phase: "response",
+ status: "failed",
+ detail: "resolved model changed mid-measurement",
+ });
+ res.writeHead(502, { "content-type": "application/json" });
+ res.end('{"error":"workbench proxy: resolved model changed mid-measurement"}');
+ return;
+ }
+ resolvedModel = observedModel;
+ }
const usage = terminal?.usage ?? usageFromSse(decoded, wireApi);
const responseEvidence: ResponseByteEvidence = {
bodyBase64: responseBytes.toString("base64"),
diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.ts
index 6e912e4da..6643a6cc8 100644
--- a/workbench/lib/measurement.ts
+++ b/workbench/lib/measurement.ts
@@ -568,6 +568,25 @@ export function weightedAxisTrials(
return { k: Math.round(results.axisScores[axis] * n), n };
}
+export function boundMeasurementFingerprints(
+ scenario: Scenario,
+ wire: {
+ systemSha256?: string | null;
+ toolsSha256?: string | null;
+ toolNames?: string[];
+ },
+): Pick<MeasurementIdentity["fingerprints"], "promptSha256" | "rubricSha256" | "wireSha256"> {
+ return {
+ promptSha256: sha256Bytes(scenario.prompt.replace(/\r\n?/g, "\n")),
+ rubricSha256: scenarioRubricSha256(scenario),
+ wireSha256: sha256Canonical({
+ systemSha256: wire.systemSha256 ?? null,
+ toolsSha256: wire.toolsSha256 ?? null,
+ toolNames: wire.toolNames ?? [],
+ }),
+ };
+}
+
function scenarioRubricSha256(scenario: Scenario): string {
return sha256Canonical({
behaviour: scenario.behaviour.map((check) => ({
@@ -738,8 +757,10 @@ export function prepareMeasurementCandidate(options: PrepareMeasurementCandidate
return artifactRef(runDir, absolute, "main-cassette");
});
const fingerprints = wireIdentity(options.results);
- fingerprints.promptSha256 = sha256Bytes(options.scenario.prompt.replace(/\r\n?/g, "\n"));
- fingerprints.rubricSha256 = scenarioRubricSha256(options.scenario);
+ const bound = boundMeasurementFingerprints(options.scenario, options.results[0]?.run.wire ?? {});
+ fingerprints.promptSha256 = bound.promptSha256;
+ fingerprints.wireSha256 = bound.wireSha256;
+ fingerprints.rubricSha256 = bound.rubricSha256;
const agentTransports = cassetteTransportProfiles(options.cassetteFiles);
if (agentTransports) {
fingerprints.transportSha256 = transportFingerprint(agentTransports, "no-judge");
@@ -981,6 +1002,20 @@ function assertRequiredRoles(manifest: MeasurementManifest): void {
}
}
+function assertBoundFingerprints(manifest: MeasurementManifest, report: WorkbenchReport): void {
+ const scenario = getScenario(manifest.scenario.id);
+ const bound = boundMeasurementFingerprints(scenario, report.fingerprint);
+ if (manifest.fingerprints.promptSha256 !== bound.promptSha256) {
+ fail("INCOMPATIBLE_IDENTITIES", "prompt fingerprint does not match the registered scenario");
+ }
+ if (manifest.fingerprints.rubricSha256 !== bound.rubricSha256) {
+ fail("INCOMPATIBLE_IDENTITIES", "rubric fingerprint does not match the registered scenario");
+ }
+ if (manifest.fingerprints.wireSha256 !== bound.wireSha256) {
+ fail("INCOMPATIBLE_IDENTITIES", "wire fingerprint does not match the bound report contract");
+ }
+}
+
function assertCountsAgainstReport(manifest: MeasurementManifest, report: WorkbenchReport): void {
const scenario = report.scenarios.find((entry) => entry.scenarioId === manifest.scenario.id);
if (!scenario) fail("CHECK_COUNT_MISMATCH", "report does not contain the measured scenario");
@@ -1231,10 +1266,9 @@ export function verifyMeasurementDirectory(
const reportRefs = roleRefs(manifest, "report");
if (reportRefs.length !== 1)
fail("CANDIDATE_INCOMPLETE", "exactly one report artifact is required");
- assertCountsAgainstReport(
- manifest,
- readJson(resolveArtifact(root, reportRefs[0].path)) as WorkbenchReport,
- );
+ const report = readJson(resolveArtifact(root, reportRefs[0].path)) as WorkbenchReport;
+ assertCountsAgainstReport(manifest, report);
+ assertBoundFingerprints(manifest, report);
const reviewRefs = roleRefs(manifest, "review-receipt");
if (reviewRefs.length !== 1)
fail("REVIEW_MISSING", "exactly one explicit review receipt is required");
@@ -1540,4 +1574,14 @@ export function verifyBoundBaseline(baseline: BoundBaseline, verified: VerifiedM
if (canonicalJson(baseline.counts) !== canonicalJson(verified.manifest.results)) {
fail("CHECK_COUNT_MISMATCH", "baseline k/n does not match its bound measurement");
}
+ const expected = boundBaselineFromMeasurement(verified);
+ if (canonicalJson(baseline.expectedFailures) !== canonicalJson(expected.expectedFailures)) {
+ fail("CHECK_COUNT_MISMATCH", "baseline expectedFailures do not match its bound measurement");
+ }
+ if (baseline.recordedAt !== expected.recordedAt) {
+ fail("INCOMPATIBLE_IDENTITIES", "baseline recordedAt does not match its measurement");
+ }
+ if (canonicalJson(baseline.identity) !== canonicalJson(expected.identity)) {
+ fail("INCOMPATIBLE_IDENTITIES", "baseline identity is not rebound from the measurement");
+ }
}
diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.ts
index c2d2e963e..4575131f3 100644
--- a/workbench/lib/runner.ts
+++ b/workbench/lib/runner.ts
@@ -12,10 +12,16 @@
// observed, not because nothing happened. That false green cost one live run to
// find; the proxy is now the only live path.
+import { existsSync } from "node:fs";
import { getReasoningCapability } from "../../electron/ai-edition/deep-agent/chat-model";
import type { LlmConfigStore } from "../../electron/ai-edition/llm-config-store";
import type { AxcutDocument } from "../../src/lib/ai-edition/schema";
-import { startRecorder } from "./cassette";
+import {
+ mergeRetryCassettes,
+ readCassetteEvidence,
+ startRecorder,
+ writeCassette,
+} from "./cassette";
import { requireLiveEnv } from "./env";
import {
DEFAULT_TURN_TIMEOUT_MS,
@@ -199,12 +205,16 @@ export async function runScenarioReps(
for (let rep = 0; rep < options.reps; rep += 1) {
budget?.refreshDeadline();
+ const canonicalFile = options.live?.record?.(rep);
+ const priorCassettes: ReturnType<typeof readCassetteEvidence>[] = [];
let attempt = 0;
for (;;) {
+ const attemptFile =
+ canonicalFile === undefined ? undefined : `${canonicalFile}.attempt-${attempt}`;
const endpoint = options.live
? await startLiveEndpoint({
scenario: options.scenario.id,
- cassetteFile: options.live.record?.(rep),
+ cassetteFile: attemptFile,
transport,
budget,
})
@@ -222,12 +232,19 @@ export async function runScenarioReps(
} finally {
endpoint?.close();
}
+ if (attemptFile && existsSync(attemptFile)) {
+ priorCassettes.push(readCassetteEvidence(attemptFile));
+ }
const failureClass = result.scored.failureClass;
if ((failureClass === "TIMEOUT" || failureClass === "TRANSPORT") && attempt < maxRetries) {
discarded.push(result);
attempt += 1;
continue;
}
+ if (canonicalFile && priorCassettes.length > 0) {
+ const success = priorCassettes[priorCassettes.length - 1];
+ writeCassette(canonicalFile, mergeRetryCassettes(success, priorCassettes.slice(0, -1)));
+ }
results.push(result);
options.onRepetition?.(result);
break;