Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fail transcript retirement checks on incomplete histories (#1268)

Regan Bellb7730aaddf3b
diff --git a/scripts/lib/tape-retirement.ts b/scripts/lib/tape-retirement.ts
index b658ad0e1..0d9989e25 100644
--- a/scripts/lib/tape-retirement.ts
+++ b/scripts/lib/tape-retirement.ts
@@ -81,11 +81,16 @@ export async function assessRenderImport(
if (latest < 0) return { action: "skip", reason: "empty" };
const coverage = await store.tapeCoverage(sessionId);
const rows = await store.getTape(sessionId);
+ const entries = await store.getEntries(sessionId);
if (!opts?.force && coverage >= latest) {
const projection = projectTapeEntries(sessionId, rows);
- if (projection && projection.coveredSeq >= latest) return { action: "skip", reason: "covered" };
+ if (
+ projection &&
+ projection.coveredSeq >= latest &&
+ classifyDivergences(entries, projection.entries, { coarse: false }).real.length === 0
+ )
+ return { action: "skip", reason: "covered" };
}
- const entries = await store.getEntries(sessionId);
if (entries.length > RENDER_IMPORT_MAX_ENTRIES) return { action: "skip", reason: "oversize" };
if (entries.some((e, i) => e.seq !== i)) return { action: "skip", reason: "gapped" };
const needsFoldImport = coverage < latest || lastImportLacksScopes(rows);
@@ -171,11 +176,10 @@ function toolPayloadCompatible(entryPayload: unknown, projectedPayload: unknown)
if (!isObj(entryPayload) || !isObj(projectedPayload)) {
return canonicalJson(entryPayload) === canonicalJson(projectedPayload);
}
+ if ((entryPayload.isError === true) !== (projectedPayload.isError === true)) return false;
+ if (Object.keys(entryPayload).some((key) => !(key in projectedPayload))) return false;
return Object.keys(projectedPayload).every(
- (k) =>
- !(k in entryPayload) ||
- canonicalJson(entryPayload[k]) === canonicalJson(projectedPayload[k]) ||
- (k === "isError" && entryPayload.isError === true && projectedPayload.isError === false),
+ (k) => !(k in entryPayload) || canonicalJson(entryPayload[k]) === canonicalJson(projectedPayload[k]),
);
}
@@ -194,7 +198,7 @@ function taintCleared(entry: SessionEntry, projected: SessionEntry): boolean {
}
function matchKey(entry: SessionEntry): string {
- return toolIdentity(entry) ?? `${entry.type}:${entry.scopeLabel}:${canonicalJson(entry.payload)}`;
+ return `${entry.scopeLabel}:${toolIdentity(entry) ?? `${entry.type}:${canonicalJson(entry.payload)}`}`;
}
export function classifyDivergences(
@@ -219,7 +223,8 @@ export function classifyDivergences(
segmentOf.set(e.seq, segment);
const p = projectedBySeq.get(e.seq);
if (!p) {
- if (opts.coarse) benign["coarse-gap"]++;
+ if (opts.coarse && (e.type === "tool_call" || e.type === "tool_result" || e.type === "thinking"))
+ benign["coarse-gap"]++;
else real.push({ seq: e.seq, field: "missing-row", entry: e, projected: undefined });
continue;
}
@@ -340,8 +345,8 @@ export async function limitedSessionParity(
const snapshotLatest = entries.length ? entries[entries.length - 1]!.seq : -1;
if (covered < snapshotLatest) return { status: "fallback" };
const served = servedRead.filter((e) => e.seq <= snapshotLatest);
- const floor = served[0]?.seq ?? Number.MAX_SAFE_INTEGER;
- const expected = entries.filter((e) => e.seq >= floor && e.seq <= covered);
+ if (servedRead.some((e) => e.seq > snapshotLatest)) return { status: "fallback" };
+ const expected = entries.filter((e) => e.seq <= covered).slice(-limit);
return {
status: "projected",
report: classifyDivergences(expected, served, { coarse: coarseTape(rows) }),
diff --git a/scripts/tape-parity-check.ts b/scripts/tape-parity-check.ts
index cd2e87298..4ef13477b 100644
--- a/scripts/tape-parity-check.ts
+++ b/scripts/tape-parity-check.ts
@@ -141,4 +141,15 @@ console.log(
`real mismatches: ${realTotal} across ${realSessions} session(s), plus ${limitedRealTotal} on limited reads`,
].join("\n"),
);
-process.exit(realTotal || limitedRealTotal || failed || vacuous ? 1 : 0);
+process.exit(
+ realTotal ||
+ limitedRealTotal ||
+ failed ||
+ vacuous ||
+ unservable.blocked ||
+ unservable.uncovered ||
+ limitedFallback ||
+ benign["coarse-gap"]
+ ? 1
+ : 0,
+);
diff --git a/test/tape-parity-gate.test.ts b/test/tape-parity-gate.test.ts
new file mode 100644
index 000000000..5a3fa8504
--- /dev/null
+++ b/test/tape-parity-gate.test.ts
@@ -0,0 +1,45 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { test } from "node:test";
+
+for (const outcome of ["clean", "blocked", "uncovered", "fallback", "coarse-gap", "failed"] as const) {
+ test(`retirement gate ${outcome === "clean" ? "accepts" : "rejects"} ${outcome}`, () => {
+ const program = `
+ import { mock } from 'node:test';
+ const outcome = ${JSON.stringify(outcome)};
+ mock.module(${JSON.stringify(new URL("../scripts/lib/backfill-runner.ts", import.meta.url).href)}, {
+ namedExports: {
+ argValue: () => undefined,
+ openSessionStore: () => ({
+ getEntries: async (id) => { if (id === 'target' && outcome === 'failed') throw new Error('read failed'); return []; },
+ getTape: async () => [],
+ }),
+ resolveSessions: async () => [{id: 'clean'}, {id: 'target'}],
+ },
+ });
+ mock.module(${JSON.stringify(new URL("../scripts/lib/tape-retirement.ts", import.meta.url).href)}, {
+ namedExports: {
+ DIVERGENCE_CLASSES: ['coarse-gap'],
+ emptyBenignCounts: () => ({'coarse-gap': 0}),
+ sessionParity: (id) => id === 'target' && ['blocked', 'uncovered'].includes(outcome)
+ ? {status: 'unservable', reason: outcome}
+ : {status: 'compared', report: {real: [], benign: {'coarse-gap': Number(id === 'target' && outcome === 'coarse-gap')}}},
+ limitedSessionParity: async (_store, id) => id === 'target' && outcome === 'fallback'
+ ? {status: 'fallback'}
+ : {status: 'projected', report: {real: []}},
+ },
+ });
+ await import(${JSON.stringify(new URL("../scripts/tape-parity-check.ts", import.meta.url).href)});
+ `;
+ const result = spawnSync(
+ process.execPath,
+ ["--experimental-test-module-mocks", "--input-type=module", "-e", program],
+ {
+ encoding: "utf8",
+ timeout: 10_000,
+ },
+ );
+ assert.equal(result.status, outcome === "clean" ? 0 : 1, result.stdout + result.stderr);
+ assert.match(result.stdout, /sessions 2: compared/);
+ });
+}
diff --git a/test/tape-retirement.test.ts b/test/tape-retirement.test.ts
index a6e1280b4..e83f611a9 100644
--- a/test/tape-retirement.test.ts
+++ b/test/tape-retirement.test.ts
@@ -214,7 +214,7 @@ test("a projection that fabricates timing the entry lacks is a real mismatch", (
assert.equal(real.length, 1);
});
-test("a curated error flag the tape never carried is the tool-payload benign class", () => {
+test("a curated error flag missing from tape blocks retirement", () => {
const entry = entryAt(3, "tool_result", {
tool: "read",
found: false,
@@ -229,8 +229,8 @@ test("a curated error flag the tape never carried is the tool-payload benign cla
result: "[no such file: x.md]",
});
const { real, benign } = classifyDivergences([entry], [projected], { coarse: false });
- assert.equal(real.length, 0);
- assert.equal(benign["tool-payload"], 1);
+ assert.equal(real.length, 1);
+ assert.equal(benign["tool-payload"], 0);
});
test("a projection inventing an error flag the entry lacks stays a real mismatch", () => {
@@ -470,7 +470,7 @@ test("classifier: identical transcripts report nothing", () => {
assert.deepEqual(report.benign, emptyBenignCounts());
});
-test("classifier: tool payload extras are benign; shared-field corruption is real", () => {
+test("classifier: projected tool extras are benign; lost fields and corruption are real", () => {
const entries = [
entryAt(0, "user", { text: "go" }),
entryAt(1, "tool_call", { tool: "execute", callId: "c1", command: "ls", chars: 120, ok: true }),
@@ -479,8 +479,14 @@ test("classifier: tool payload extras are benign; shared-field corruption is rea
];
const extras = [
entries[0]!,
- { ...entries[1]!, payload: { command: "ls", tool: "execute", callId: "c1", text: "retained-arg" } },
- { ...entries[2]!, payload: { tool: "execute", callId: "c1", isError: false, result: "a.txt\nb.txt" } },
+ {
+ ...entries[1]!,
+ payload: { command: "ls", tool: "execute", callId: "c1", chars: 120, ok: true, text: "retained-arg" },
+ },
+ {
+ ...entries[2]!,
+ payload: { tool: "execute", callId: "c1", isError: false, result: "a.txt\nb.txt", files: 2, extra: true },
+ },
entries[3]!,
];
const benignReport = classifyDivergences(entries, extras, { coarse: false });
@@ -676,7 +682,78 @@ test("limitedSessionParity exercises the bounded read path and detects fallback"
const staleRows = await sim.store.getTape(sim.session.id);
await simLiveTurn(sim, { input: "one more thing", ts: "1720000000.000300", reply: "Done." });
const raced = await limitedSessionParity(sim.store, sim.session.id, staleEntries, staleRows, 3);
- assert.equal(raced.status, "projected");
- assert.ok(raced.status === "projected");
- assert.deepEqual(raced.report.real, [], "a turn landing between the snapshot and the serving read is not a mismatch");
+ assert.deepEqual(raced, { status: "fallback" });
+});
+
+test("classifier rejects changed tool scopes, lost results, and cleared errors", () => {
+ const entry = entryAt(0, "tool_result", { tool: "execute", callId: "c1", result: "failed", isError: true });
+ for (const projected of [
+ { ...entry, scopeLabel: "org:public" as ScopeId },
+ { ...entry, payload: { tool: "execute", callId: "c1", isError: true } },
+ { ...entry, payload: { ...(entry.payload as object), isError: false } },
+ ]) {
+ assert.equal(classifyDivergences([entry], [projected], { coarse: false }).real.length, 1);
+ }
+});
+
+test("coarse transcripts must retain user, assistant and system rows", () => {
+ for (const type of ["user", "assistant", "system"] as const) {
+ assert.equal(classifyDivergences([entryAt(0, type, { text: "retain" })], [], { coarse: true }).real.length, 1);
+ }
+});
+
+test("a covered tape with lost payload is repaired without a forced import", async () => {
+ const sim = await preCutoverSession();
+ await importSession(sim);
+ const entries = await sim.store.getEntries(sim.session.id);
+ const changed = entries.map((entry) =>
+ entry.type === "assistant"
+ ? { ...entry, payload: { ...(entry.payload as object), durableDetail: "must survive retirement" } }
+ : entry,
+ );
+ const plan = await assessRenderImport(
+ {
+ latestEntrySeq: (id) => sim.store.latestEntrySeq(id),
+ tapeCoverage: (id) => sim.store.tapeCoverage(id),
+ getTape: (id) => sim.store.getTape(id),
+ getEntries: async () => changed,
+ },
+ sim.session.id,
+ );
+ assert.equal(plan.action, "import");
+ assert.ok(plan.action === "import");
+ assert.equal(plan.needsFoldImport, false);
+ await appendRenderImport(sim.store, sim.lease, plan.entries, scope, plan.needsFoldImport);
+ const parity = sessionParity(sim.session.id, changed, await sim.store.getTape(sim.session.id));
+ assert.ok(parity.status === "compared");
+ assert.deepEqual(parity.report.real, []);
+});
+
+test("an absent error flag cannot become a projected failure", () => {
+ const entry = entryAt(0, "tool_result", { tool: "execute", callId: "c1", result: "ok" });
+ const projected = { ...entry, payload: { ...(entry.payload as object), isError: true } };
+ assert.equal(classifyDivergences([entry], [projected], { coarse: false }).real.length, 1);
+});
+
+test("limited parity rejects an empty served suffix even when coverage is complete", async () => {
+ const sim = await preCutoverSession();
+ await importSession(sim);
+ const entries = await sim.store.getEntries(sim.session.id);
+ const rows = await sim.store.getTape(sim.session.id);
+ const result = await limitedSessionParity(
+ {
+ getEntries: (id, opts) => sim.store.getEntries(id, opts),
+ visibleEntries: (id, principal) => sim.store.visibleEntries(id, principal),
+ getTape: (id, opts) => sim.store.getTape(id, opts),
+ latestEntrySeq: async () => -1,
+ participantWindowsOf: (id) => sim.store.participantWindowsOf(id),
+ },
+ sim.session.id,
+ entries,
+ rows,
+ 3,
+ );
+ assert.ok(result.status === "projected");
+ assert.equal(result.report.real.length, 3);
+ assert.ok(result.report.real.every((row) => row.field === "missing-row"));
});