Multiplayer agent harness for work.
Fail transcript retirement checks on incomplete histories (#1268)
Regan Bellb7730aaddf3b
diff --git a/scripts/lib/tape-retirement.ts b/scripts/lib/tape-retirement.tsindex b658ad0e1..0d9989e25 100644--- a/scripts/lib/tape-retirement.ts+++ b/scripts/lib/tape-retirement.ts@@ -81,11 +81,16 @@ export async function assessRenderImport(if (latest < 0) return { action: "skip", reason: "empty" };const coverage = await store.tapeCoverage(sessionId);const rows = await store.getTape(sessionId);+ const entries = await store.getEntries(sessionId);if (!opts?.force && coverage >= latest) {const projection = projectTapeEntries(sessionId, rows);- if (projection && projection.coveredSeq >= latest) return { action: "skip", reason: "covered" };+ if (+ projection &&+ projection.coveredSeq >= latest &&+ classifyDivergences(entries, projection.entries, { coarse: false }).real.length === 0+ )+ return { action: "skip", reason: "covered" };}- const entries = await store.getEntries(sessionId);if (entries.length > RENDER_IMPORT_MAX_ENTRIES) return { action: "skip", reason: "oversize" };if (entries.some((e, i) => e.seq !== i)) return { action: "skip", reason: "gapped" };const needsFoldImport = coverage < latest || lastImportLacksScopes(rows);@@ -171,11 +176,10 @@ function toolPayloadCompatible(entryPayload: unknown, projectedPayload: unknown)if (!isObj(entryPayload) || !isObj(projectedPayload)) {return canonicalJson(entryPayload) === canonicalJson(projectedPayload);}+ if ((entryPayload.isError === true) !== (projectedPayload.isError === true)) return false;+ if (Object.keys(entryPayload).some((key) => !(key in projectedPayload))) return false;return Object.keys(projectedPayload).every(- (k) =>- !(k in entryPayload) ||- canonicalJson(entryPayload[k]) === canonicalJson(projectedPayload[k]) ||- (k === "isError" && entryPayload.isError === true && projectedPayload.isError === false),+ (k) => !(k in entryPayload) || canonicalJson(entryPayload[k]) === canonicalJson(projectedPayload[k]),);}@@ -194,7 +198,7 @@ function taintCleared(entry: SessionEntry, projected: SessionEntry): boolean {}function matchKey(entry: SessionEntry): string {- return toolIdentity(entry) ?? `${entry.type}:${entry.scopeLabel}:${canonicalJson(entry.payload)}`;+ return `${entry.scopeLabel}:${toolIdentity(entry) ?? `${entry.type}:${canonicalJson(entry.payload)}`}`;}export function classifyDivergences(@@ -219,7 +223,8 @@ export function classifyDivergences(segmentOf.set(e.seq, segment);const p = projectedBySeq.get(e.seq);if (!p) {- if (opts.coarse) benign["coarse-gap"]++;+ if (opts.coarse && (e.type === "tool_call" || e.type === "tool_result" || e.type === "thinking"))+ benign["coarse-gap"]++;else real.push({ seq: e.seq, field: "missing-row", entry: e, projected: undefined });continue;}@@ -340,8 +345,8 @@ export async function limitedSessionParity(const snapshotLatest = entries.length ? entries[entries.length - 1]!.seq : -1;if (covered < snapshotLatest) return { status: "fallback" };const served = servedRead.filter((e) => e.seq <= snapshotLatest);- const floor = served[0]?.seq ?? Number.MAX_SAFE_INTEGER;- const expected = entries.filter((e) => e.seq >= floor && e.seq <= covered);+ if (servedRead.some((e) => e.seq > snapshotLatest)) return { status: "fallback" };+ const expected = entries.filter((e) => e.seq <= covered).slice(-limit);return {status: "projected",report: classifyDivergences(expected, served, { coarse: coarseTape(rows) }),diff --git a/scripts/tape-parity-check.ts b/scripts/tape-parity-check.tsindex cd2e87298..4ef13477b 100644--- a/scripts/tape-parity-check.ts+++ b/scripts/tape-parity-check.ts@@ -141,4 +141,15 @@ console.log(`real mismatches: ${realTotal} across ${realSessions} session(s), plus ${limitedRealTotal} on limited reads`,].join("\n"),);-process.exit(realTotal || limitedRealTotal || failed || vacuous ? 1 : 0);+process.exit(+ realTotal ||+ limitedRealTotal ||+ failed ||+ vacuous ||+ unservable.blocked ||+ unservable.uncovered ||+ limitedFallback ||+ benign["coarse-gap"]+ ? 1+ : 0,+);diff --git a/test/tape-parity-gate.test.ts b/test/tape-parity-gate.test.tsnew file mode 100644index 000000000..5a3fa8504--- /dev/null+++ b/test/tape-parity-gate.test.ts@@ -0,0 +1,45 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import { test } from "node:test";++for (const outcome of ["clean", "blocked", "uncovered", "fallback", "coarse-gap", "failed"] as const) {+ test(`retirement gate ${outcome === "clean" ? "accepts" : "rejects"} ${outcome}`, () => {+ const program = `+ import { mock } from 'node:test';+ const outcome = ${JSON.stringify(outcome)};+ mock.module(${JSON.stringify(new URL("../scripts/lib/backfill-runner.ts", import.meta.url).href)}, {+ namedExports: {+ argValue: () => undefined,+ openSessionStore: () => ({+ getEntries: async (id) => { if (id === 'target' && outcome === 'failed') throw new Error('read failed'); return []; },+ getTape: async () => [],+ }),+ resolveSessions: async () => [{id: 'clean'}, {id: 'target'}],+ },+ });+ mock.module(${JSON.stringify(new URL("../scripts/lib/tape-retirement.ts", import.meta.url).href)}, {+ namedExports: {+ DIVERGENCE_CLASSES: ['coarse-gap'],+ emptyBenignCounts: () => ({'coarse-gap': 0}),+ sessionParity: (id) => id === 'target' && ['blocked', 'uncovered'].includes(outcome)+ ? {status: 'unservable', reason: outcome}+ : {status: 'compared', report: {real: [], benign: {'coarse-gap': Number(id === 'target' && outcome === 'coarse-gap')}}},+ limitedSessionParity: async (_store, id) => id === 'target' && outcome === 'fallback'+ ? {status: 'fallback'}+ : {status: 'projected', report: {real: []}},+ },+ });+ await import(${JSON.stringify(new URL("../scripts/tape-parity-check.ts", import.meta.url).href)});+ `;+ const result = spawnSync(+ process.execPath,+ ["--experimental-test-module-mocks", "--input-type=module", "-e", program],+ {+ encoding: "utf8",+ timeout: 10_000,+ },+ );+ assert.equal(result.status, outcome === "clean" ? 0 : 1, result.stdout + result.stderr);+ assert.match(result.stdout, /sessions 2: compared/);+ });+}diff --git a/test/tape-retirement.test.ts b/test/tape-retirement.test.tsindex a6e1280b4..e83f611a9 100644--- a/test/tape-retirement.test.ts+++ b/test/tape-retirement.test.ts@@ -214,7 +214,7 @@ test("a projection that fabricates timing the entry lacks is a real mismatch", (assert.equal(real.length, 1);});-test("a curated error flag the tape never carried is the tool-payload benign class", () => {+test("a curated error flag missing from tape blocks retirement", () => {const entry = entryAt(3, "tool_result", {tool: "read",found: false,@@ -229,8 +229,8 @@ test("a curated error flag the tape never carried is the tool-payload benign claresult: "[no such file: x.md]",});const { real, benign } = classifyDivergences([entry], [projected], { coarse: false });- assert.equal(real.length, 0);- assert.equal(benign["tool-payload"], 1);+ assert.equal(real.length, 1);+ assert.equal(benign["tool-payload"], 0);});test("a projection inventing an error flag the entry lacks stays a real mismatch", () => {@@ -470,7 +470,7 @@ test("classifier: identical transcripts report nothing", () => {assert.deepEqual(report.benign, emptyBenignCounts());});-test("classifier: tool payload extras are benign; shared-field corruption is real", () => {+test("classifier: projected tool extras are benign; lost fields and corruption are real", () => {const entries = [entryAt(0, "user", { text: "go" }),entryAt(1, "tool_call", { tool: "execute", callId: "c1", command: "ls", chars: 120, ok: true }),@@ -479,8 +479,14 @@ test("classifier: tool payload extras are benign; shared-field corruption is rea];const extras = [entries[0]!,- { ...entries[1]!, payload: { command: "ls", tool: "execute", callId: "c1", text: "retained-arg" } },- { ...entries[2]!, payload: { tool: "execute", callId: "c1", isError: false, result: "a.txt\nb.txt" } },+ {+ ...entries[1]!,+ payload: { command: "ls", tool: "execute", callId: "c1", chars: 120, ok: true, text: "retained-arg" },+ },+ {+ ...entries[2]!,+ payload: { tool: "execute", callId: "c1", isError: false, result: "a.txt\nb.txt", files: 2, extra: true },+ },entries[3]!,];const benignReport = classifyDivergences(entries, extras, { coarse: false });@@ -676,7 +682,78 @@ test("limitedSessionParity exercises the bounded read path and detects fallback"const staleRows = await sim.store.getTape(sim.session.id);await simLiveTurn(sim, { input: "one more thing", ts: "1720000000.000300", reply: "Done." });const raced = await limitedSessionParity(sim.store, sim.session.id, staleEntries, staleRows, 3);- assert.equal(raced.status, "projected");- assert.ok(raced.status === "projected");- assert.deepEqual(raced.report.real, [], "a turn landing between the snapshot and the serving read is not a mismatch");+ assert.deepEqual(raced, { status: "fallback" });+});++test("classifier rejects changed tool scopes, lost results, and cleared errors", () => {+ const entry = entryAt(0, "tool_result", { tool: "execute", callId: "c1", result: "failed", isError: true });+ for (const projected of [+ { ...entry, scopeLabel: "org:public" as ScopeId },+ { ...entry, payload: { tool: "execute", callId: "c1", isError: true } },+ { ...entry, payload: { ...(entry.payload as object), isError: false } },+ ]) {+ assert.equal(classifyDivergences([entry], [projected], { coarse: false }).real.length, 1);+ }+});++test("coarse transcripts must retain user, assistant and system rows", () => {+ for (const type of ["user", "assistant", "system"] as const) {+ assert.equal(classifyDivergences([entryAt(0, type, { text: "retain" })], [], { coarse: true }).real.length, 1);+ }+});++test("a covered tape with lost payload is repaired without a forced import", async () => {+ const sim = await preCutoverSession();+ await importSession(sim);+ const entries = await sim.store.getEntries(sim.session.id);+ const changed = entries.map((entry) =>+ entry.type === "assistant"+ ? { ...entry, payload: { ...(entry.payload as object), durableDetail: "must survive retirement" } }+ : entry,+ );+ const plan = await assessRenderImport(+ {+ latestEntrySeq: (id) => sim.store.latestEntrySeq(id),+ tapeCoverage: (id) => sim.store.tapeCoverage(id),+ getTape: (id) => sim.store.getTape(id),+ getEntries: async () => changed,+ },+ sim.session.id,+ );+ assert.equal(plan.action, "import");+ assert.ok(plan.action === "import");+ assert.equal(plan.needsFoldImport, false);+ await appendRenderImport(sim.store, sim.lease, plan.entries, scope, plan.needsFoldImport);+ const parity = sessionParity(sim.session.id, changed, await sim.store.getTape(sim.session.id));+ assert.ok(parity.status === "compared");+ assert.deepEqual(parity.report.real, []);+});++test("an absent error flag cannot become a projected failure", () => {+ const entry = entryAt(0, "tool_result", { tool: "execute", callId: "c1", result: "ok" });+ const projected = { ...entry, payload: { ...(entry.payload as object), isError: true } };+ assert.equal(classifyDivergences([entry], [projected], { coarse: false }).real.length, 1);+});++test("limited parity rejects an empty served suffix even when coverage is complete", async () => {+ const sim = await preCutoverSession();+ await importSession(sim);+ const entries = await sim.store.getEntries(sim.session.id);+ const rows = await sim.store.getTape(sim.session.id);+ const result = await limitedSessionParity(+ {+ getEntries: (id, opts) => sim.store.getEntries(id, opts),+ visibleEntries: (id, principal) => sim.store.visibleEntries(id, principal),+ getTape: (id, opts) => sim.store.getTape(id, opts),+ latestEntrySeq: async () => -1,+ participantWindowsOf: (id) => sim.store.participantWindowsOf(id),+ },+ sim.session.id,+ entries,+ rows,+ 3,+ );+ assert.ok(result.status === "projected");+ assert.equal(result.report.real.length, 3);+ assert.ok(result.report.real.every((row) => row.field === "missing-row"));});