Record your screen, ship a demo. Free and open-source, GPU-accelerated, no watermarks, no subscriptions. Windows, macOS, Linux. Actively maintained.
TypeScript cross-platformelectronffmpegopen-sourceproduct-demorustscreen-capturescreen-recorderscreen-recordingscreen-studioscreencastspeech-to-texttypescriptvideo-editingvideo-editorwaylandwgpuwhisper
fix(workbench): bind retry evidence, fingerprints, and Chat model identity
超超超超超级喜欢你的达妮娅70ec779b97a8
diff --git a/workbench/cli.ts b/workbench/cli.tsindex 04fa926c9..b862690ed 100644--- a/workbench/cli.ts+++ b/workbench/cli.ts@@ -569,7 +569,6 @@ async function commandJudge(options: Options): Promise<number> {if (!liveJudgeTransport || !liveJudgeBudget) {throw new Error("live judge transport was not prepared");}- liveJudgeBudget.refreshDeadline();endpoint = await startRecorder({// Le proxy, pas le provider en direct — même règle que `runner.ts` :// c'est le seul endroit d'où une cassette peut sortir, et le seul@@ -612,6 +611,7 @@ async function commandJudge(options: Options): Promise<number> {const context = contextFromPersistedTurn(turn);const readings = new Map<string, JudgeReading>();for (const judged of scenario.judged ?? []) {+ liveJudgeBudget?.refreshDeadline();const reading = await askJudge({endpoint: {baseUrl: endpoint.url,diff --git a/workbench/l0/measurements.wb.ts b/workbench/l0/measurements.wb.tsindex 93a9c3059..f9706d48f 100644--- a/workbench/l0/measurements.wb.ts+++ b/workbench/l0/measurements.wb.ts@@ -15,6 +15,7 @@ import {type ArtifactReference,assertComparable,boundBaselineFromMeasurement,+ boundMeasurementFingerprints,exportMeasurement,MeasurementError,type MeasurementManifest,@@ -33,6 +34,7 @@ import {} from "../lib/provenance";import { wilson95 } from "../lib/stats";import { runMeasurementCli } from "../measurement-cli";+import { getScenario } from "../scenarios/registry";const roots: string[] = [];const cliOutputs: string[] = [];@@ -186,11 +188,13 @@ function createCandidate(root: string): {telemetrySha256: fixture.telemetrySha256,},fingerprints: {- promptSha256: HASH("1"),+ ...boundMeasurementFingerprints(getScenario("target-right-clip"), {+ systemSha256: HASH("d"),+ toolsSha256: HASH("e"),+ toolNames: [],+ }),systemSha256: HASH("d"),toolsSha256: HASH("e"),- wireSha256: HASH("4"),- rubricSha256: HASH("5"),},models: {agent: { requested: "loopback-model", observed: "loopback-model" },@@ -661,9 +665,30 @@ describe("measurement identity and bound baseline", () => {const changedSource = structuredClone(baseline);changedSource.effectiveSourceSha256 = HASH("0");expectCode(() => verifyBoundBaseline(changedSource, verified), "SOURCE_IDENTITY_MISMATCH");+ const changedFailures = structuredClone(baseline);+ changedFailures.expectedFailures = ["whatever-I-want"];+ expectCode(() => verifyBoundBaseline(changedFailures, verified), "CHECK_COUNT_MISMATCH");expect(() => assertComparable(baseline.identity, verified.manifest)).not.toThrow();});+ it("rejects a rewritten prompt, wire, or rubric fingerprint", () => {+ const fingerprints = createCandidate(root());+ rewriteCandidate(fingerprints.runDir, (manifest) => {+ manifest.fingerprints.promptSha256 = HASH("1");+ });+ expectCode(() => exportValid(fingerprints), "INCOMPATIBLE_IDENTITIES");+ const wire = createCandidate(root());+ rewriteCandidate(wire.runDir, (manifest) => {+ manifest.fingerprints.wireSha256 = HASH("4");+ });+ expectCode(() => exportValid(wire), "INCOMPATIBLE_IDENTITIES");+ const rubric = createCandidate(root());+ rewriteCandidate(rubric.runDir, (manifest) => {+ manifest.fingerprints.rubricSha256 = HASH("5");+ });+ expectCode(() => exportValid(rubric), "INCOMPATIBLE_IDENTITIES");+ });+it("never turns an explicit rejected review into baseline approval", () => {const base = createCandidate(root());const receipt = JSON.parse(readFileSync(base.review, "utf8")) as { disposition: string };diff --git a/workbench/l1/cassette.wb.ts b/workbench/l1/cassette.wb.tsindex 73f5aac70..799a9f118 100644--- a/workbench/l1/cassette.wb.ts+++ b/workbench/l1/cassette.wb.ts@@ -13,6 +13,7 @@ import { join } from "node:path";import { afterAll, describe, expect, it } from "vitest";import {hashRequest,+ mergeRetryCassettes,readCassette,startRecorder,startReplay,@@ -489,4 +490,110 @@ describe("record then replay", () => {target.close();}});++ it("keeps failed retry attempts on the successful cassette", () => {+ const digest = { systemChars: 1, toolCount: 0, roles: ["user"], lastUserText: "retry" };+ const failed = {+ scenario: "retry-probe",+ provider: "loopback",+ model: "loopback",+ recordedAt: "2026-09-13T00:00:00.000Z",+ rounds: [+ {+ round: 0,+ requestHash: "0123456789abcdef",+ digest,+ sse: "data: [DONE]\n\n",+ },+ ],+ attempts: [+ { attempt: 0, phase: "forward" as const, status: "failed" as const, detail: "timeout" },+ ],+ };+ const success = {+ ...failed,+ attempts: [{ attempt: 0, phase: "complete" as const, status: "recorded" as const }],+ };+ const merged = mergeRetryCassettes(success, [failed]);+ expect(merged.rounds).toEqual(success.rounds);+ expect(merged.attempts).toEqual([+ { attempt: 0, phase: "forward", status: "failed", detail: "timeout" },+ { attempt: 1, phase: "complete", status: "recorded" },+ ]);+ });++ it("rejects a Chat cassette whose rounds resolve to different models", () => {+ expect(() =>+ writeCassette(join(DIRECTORY, "mixed-models.json"), {+ scenario: "mixed-models",+ provider: "loopback",+ model: "alias",+ resolvedModel: "gpt-a",+ recordedAt: "2026-09-13T00:00:00.000Z",+ rounds: [+ {+ round: 0,+ requestHash: "0123456789abcdef",+ digest: { systemChars: 0, toolCount: 0, roles: ["user"], lastUserText: "a" },+ sse: `data: ${JSON.stringify({ model: "gpt-a" })}\n\ndata: [DONE]\n\n`,+ },+ {+ round: 1,+ requestHash: "fedcba9876543210",+ digest: { systemChars: 0, toolCount: 0, roles: ["user"], lastUserText: "b" },+ sse: `data: ${JSON.stringify({ model: "gpt-b" })}\n\ndata: [DONE]\n\n`,+ },+ ],+ }),+ ).toThrow(/model mismatch/);+ });++ it("fails closed when the concrete Chat model changes mid-measurement", async () => {+ let round = 0;+ const upstream = createServer((_req, res) => {+ const model = round === 0 ? "gpt-a" : "gpt-b";+ round += 1;+ res.writeHead(200, { "content-type": "text/event-stream" });+ res.end(`data: ${JSON.stringify({ model })}\n\ndata: [DONE]\n\n`);+ });+ await new Promise<void>((resolve) => upstream.listen(0, "127.0.0.1", resolve));+ const address = upstream.address();+ if (!address || typeof address === "string")+ throw new Error("model-change upstream has no address");+ const file = join(DIRECTORY, "model-change.json");+ const recorder = await startRecorder({+ upstream: `http://127.0.0.1:${address.port}`,+ file,+ scenario: "model-change",+ provider: "loopback",+ model: "alias",+ });+ try {+ const first = await fetch(`${recorder.url}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ model: "alias",+ messages: [{ role: "user", content: "first" }],+ }),+ });+ expect(first.status).toBe(200);+ await first.text();+ const second = await fetch(`${recorder.url}/chat/completions`, {+ method: "POST",+ headers: { "content-type": "application/json" },+ body: JSON.stringify({+ model: "alias",+ messages: [{ role: "user", content: "second" }],+ }),+ });+ expect(second.status).toBe(502);+ expect(await second.json()).toEqual({+ error: "workbench proxy: resolved model changed mid-measurement",+ });+ } finally {+ recorder.close();+ upstream.close();+ }+ });});diff --git a/workbench/lib/cassette.ts b/workbench/lib/cassette.tsindex 3b574f8e2..ae7cbd659 100644--- a/workbench/lib/cassette.ts+++ b/workbench/lib/cassette.ts@@ -345,6 +345,7 @@ export function validateCassette(cassette: Cassette): void {"cassette has transport or Responses evidence without a wireApi discriminator",);}+ assertChatResolvedModel(cassette);return;}if (declaredWireApi !== "chat-completions" && declaredWireApi !== "responses") {@@ -369,6 +370,7 @@ export function validateCassette(cassette: Cassette): void {if (hasResponsesEvidence || hasResponsesSseEvents) {throw new Error("Chat cassette contains Responses-only round evidence");}+ assertChatResolvedModel(cassette);return;}for (const round of cassette.rounds) {@@ -399,12 +401,37 @@ export function validateCassette(cassette: Cassette): void {}}+function assertChatResolvedModel(cassette: Cassette): void {+ for (const round of cassette.rounds) {+ const model = modelFromSse(round.sse, "chat-completions");+ if (model && cassette.resolvedModel && model !== cassette.resolvedModel) {+ throw new Error(`Chat round ${round.round} model mismatch`);+ }+ }+}+export function readCassette(file: string): Cassette {const cassette = JSON.parse(readFileSync(file, "utf8")) as Cassette;validateCassette(cassette);return cassette;}+export function mergeRetryCassettes(success: Cassette, prior: readonly Cassette[]): Cassette {+ const attempts = [+ ...prior.flatMap((cassette) => cassette.attempts ?? []),+ ...(success.attempts ?? []),+ ].map((attempt, index) => ({ ...attempt, attempt: index }));+ return { ...success, attempts };+}++export function readCassetteEvidence(file: string): Cassette {+ try {+ return readCassette(file);+ } catch {+ return JSON.parse(readFileSync(file, "utf8")) as Cassette;+ }+}+export function writeCassette(file: string, cassette: Cassette): void {validateCassette(cassette);const payload = `${JSON.stringify(cassette, null, "\t")}\n`;@@ -695,7 +722,21 @@ export async function startRecorder(options: {res.end('{"error":"workbench proxy: invalid Responses terminal sequence"}');return;}- resolvedModel ??= terminal?.model ?? modelFromSse(decoded, wireApi);+ const observedModel = terminal?.model ?? modelFromSse(decoded, wireApi);+ if (observedModel) {+ if (resolvedModel && observedModel !== resolvedModel) {+ retainAttempt({+ attempt,+ phase: "response",+ status: "failed",+ detail: "resolved model changed mid-measurement",+ });+ res.writeHead(502, { "content-type": "application/json" });+ res.end('{"error":"workbench proxy: resolved model changed mid-measurement"}');+ return;+ }+ resolvedModel = observedModel;+ }const usage = terminal?.usage ?? usageFromSse(decoded, wireApi);const responseEvidence: ResponseByteEvidence = {bodyBase64: responseBytes.toString("base64"),diff --git a/workbench/lib/measurement.ts b/workbench/lib/measurement.tsindex 6e912e4da..6643a6cc8 100644--- a/workbench/lib/measurement.ts+++ b/workbench/lib/measurement.ts@@ -568,6 +568,25 @@ export function weightedAxisTrials(return { k: Math.round(results.axisScores[axis] * n), n };}+export function boundMeasurementFingerprints(+ scenario: Scenario,+ wire: {+ systemSha256?: string | null;+ toolsSha256?: string | null;+ toolNames?: string[];+ },+): Pick<MeasurementIdentity["fingerprints"], "promptSha256" | "rubricSha256" | "wireSha256"> {+ return {+ promptSha256: sha256Bytes(scenario.prompt.replace(/\r\n?/g, "\n")),+ rubricSha256: scenarioRubricSha256(scenario),+ wireSha256: sha256Canonical({+ systemSha256: wire.systemSha256 ?? null,+ toolsSha256: wire.toolsSha256 ?? null,+ toolNames: wire.toolNames ?? [],+ }),+ };+}+function scenarioRubricSha256(scenario: Scenario): string {return sha256Canonical({behaviour: scenario.behaviour.map((check) => ({@@ -738,8 +757,10 @@ export function prepareMeasurementCandidate(options: PrepareMeasurementCandidatereturn artifactRef(runDir, absolute, "main-cassette");});const fingerprints = wireIdentity(options.results);- fingerprints.promptSha256 = sha256Bytes(options.scenario.prompt.replace(/\r\n?/g, "\n"));- fingerprints.rubricSha256 = scenarioRubricSha256(options.scenario);+ const bound = boundMeasurementFingerprints(options.scenario, options.results[0]?.run.wire ?? {});+ fingerprints.promptSha256 = bound.promptSha256;+ fingerprints.wireSha256 = bound.wireSha256;+ fingerprints.rubricSha256 = bound.rubricSha256;const agentTransports = cassetteTransportProfiles(options.cassetteFiles);if (agentTransports) {fingerprints.transportSha256 = transportFingerprint(agentTransports, "no-judge");@@ -981,6 +1002,20 @@ function assertRequiredRoles(manifest: MeasurementManifest): void {}}+function assertBoundFingerprints(manifest: MeasurementManifest, report: WorkbenchReport): void {+ const scenario = getScenario(manifest.scenario.id);+ const bound = boundMeasurementFingerprints(scenario, report.fingerprint);+ if (manifest.fingerprints.promptSha256 !== bound.promptSha256) {+ fail("INCOMPATIBLE_IDENTITIES", "prompt fingerprint does not match the registered scenario");+ }+ if (manifest.fingerprints.rubricSha256 !== bound.rubricSha256) {+ fail("INCOMPATIBLE_IDENTITIES", "rubric fingerprint does not match the registered scenario");+ }+ if (manifest.fingerprints.wireSha256 !== bound.wireSha256) {+ fail("INCOMPATIBLE_IDENTITIES", "wire fingerprint does not match the bound report contract");+ }+}+function assertCountsAgainstReport(manifest: MeasurementManifest, report: WorkbenchReport): void {const scenario = report.scenarios.find((entry) => entry.scenarioId === manifest.scenario.id);if (!scenario) fail("CHECK_COUNT_MISMATCH", "report does not contain the measured scenario");@@ -1231,10 +1266,9 @@ export function verifyMeasurementDirectory(const reportRefs = roleRefs(manifest, "report");if (reportRefs.length !== 1)fail("CANDIDATE_INCOMPLETE", "exactly one report artifact is required");- assertCountsAgainstReport(- manifest,- readJson(resolveArtifact(root, reportRefs[0].path)) as WorkbenchReport,- );+ const report = readJson(resolveArtifact(root, reportRefs[0].path)) as WorkbenchReport;+ assertCountsAgainstReport(manifest, report);+ assertBoundFingerprints(manifest, report);const reviewRefs = roleRefs(manifest, "review-receipt");if (reviewRefs.length !== 1)fail("REVIEW_MISSING", "exactly one explicit review receipt is required");@@ -1540,4 +1574,14 @@ export function verifyBoundBaseline(baseline: BoundBaseline, verified: VerifiedMif (canonicalJson(baseline.counts) !== canonicalJson(verified.manifest.results)) {fail("CHECK_COUNT_MISMATCH", "baseline k/n does not match its bound measurement");}+ const expected = boundBaselineFromMeasurement(verified);+ if (canonicalJson(baseline.expectedFailures) !== canonicalJson(expected.expectedFailures)) {+ fail("CHECK_COUNT_MISMATCH", "baseline expectedFailures do not match its bound measurement");+ }+ if (baseline.recordedAt !== expected.recordedAt) {+ fail("INCOMPATIBLE_IDENTITIES", "baseline recordedAt does not match its measurement");+ }+ if (canonicalJson(baseline.identity) !== canonicalJson(expected.identity)) {+ fail("INCOMPATIBLE_IDENTITIES", "baseline identity is not rebound from the measurement");+ }}diff --git a/workbench/lib/runner.ts b/workbench/lib/runner.tsindex c2d2e963e..4575131f3 100644--- a/workbench/lib/runner.ts+++ b/workbench/lib/runner.ts@@ -12,10 +12,16 @@// observed, not because nothing happened. That false green cost one live run to// find; the proxy is now the only live path.+import { existsSync } from "node:fs";import { getReasoningCapability } from "../../electron/ai-edition/deep-agent/chat-model";import type { LlmConfigStore } from "../../electron/ai-edition/llm-config-store";import type { AxcutDocument } from "../../src/lib/ai-edition/schema";-import { startRecorder } from "./cassette";+import {+ mergeRetryCassettes,+ readCassetteEvidence,+ startRecorder,+ writeCassette,+} from "./cassette";import { requireLiveEnv } from "./env";import {DEFAULT_TURN_TIMEOUT_MS,@@ -199,12 +205,16 @@ export async function runScenarioReps(for (let rep = 0; rep < options.reps; rep += 1) {budget?.refreshDeadline();+ const canonicalFile = options.live?.record?.(rep);+ const priorCassettes: ReturnType<typeof readCassetteEvidence>[] = [];let attempt = 0;for (;;) {+ const attemptFile =+ canonicalFile === undefined ? undefined : `${canonicalFile}.attempt-${attempt}`;const endpoint = options.live? await startLiveEndpoint({scenario: options.scenario.id,- cassetteFile: options.live.record?.(rep),+ cassetteFile: attemptFile,transport,budget,})@@ -222,12 +232,19 @@ export async function runScenarioReps(} finally {endpoint?.close();}+ if (attemptFile && existsSync(attemptFile)) {+ priorCassettes.push(readCassetteEvidence(attemptFile));+ }const failureClass = result.scored.failureClass;if ((failureClass === "TIMEOUT" || failureClass === "TRANSPORT") && attempt < maxRetries) {discarded.push(result);attempt += 1;continue;}+ if (canonicalFile && priorCassettes.length > 0) {+ const success = priorCassettes[priorCassettes.length - 1];+ writeCassette(canonicalFile, mergeRetryCassettes(success, priorCassettes.slice(0, -1)));+ }results.push(result);options.onRepetition?.(result);break;