Multiplayer agent harness for work.
Preserve exact transcripts on the session tape (#1272)
Regan Bell45be37eeae6f
diff --git a/deploy/core/Dockerfile b/deploy/core/Dockerfileindex b811dc97d..f7d154836 100644--- a/deploy/core/Dockerfile+++ b/deploy/core/Dockerfile@@ -17,6 +17,8 @@ RUN npm ci --omit=dev && npm audit --omit=dev --audit-level=moderate \&& rm -rf /root/.npm /tmp/node-compile-cacheCOPY src ./src+COPY scripts/migrate-transcript-tape.ts ./scripts/migrate-transcript-tape.ts+COPY scripts/lib/transcript-tape-migration.ts ./scripts/lib/transcript-tape-migration.tsCOPY cli/templates/slack-manifest.json ./cli/templates/slack-manifest.jsonCOPY skills-seed ./skills-seedCOPY plugins/onboarding ./plugins/onboardingdiff --git a/package.json b/package.jsonindex 39ec066a5..fd1adff2b 100644--- a/package.json+++ b/package.json@@ -50,7 +50,8 @@"lint": "eslint .","lint:ox": "oxlint --deny-warnings src plugins scripts cli test","dev-instance:doctor": "bash scripts/dev-instance.sh doctor",- "lint:knip": "knip"+ "lint:knip": "knip",+ "migrate:transcript-tape": "node scripts/migrate-transcript-tape.ts"},"lint-staged": {"*": "prettier --write --ignore-unknown"diff --git a/scripts/lib/transcript-tape-migration.ts b/scripts/lib/transcript-tape-migration.tsnew file mode 100644index 000000000..eb51d9b49--- /dev/null+++ b/scripts/lib/transcript-tape-migration.ts@@ -0,0 +1,84 @@+import type { Client } from "pg";+import { rowToEntry } from "../../src/sessions/postgres-session-store.ts";+import { tapeTranscriptEntryRecord } from "../../src/sessions/session-store.ts";+import { canonicalJson } from "../../src/util/objects.ts";+import { jsonbSafeStringify } from "../../src/util/text.ts";++export async function migrateTranscriptPage(+ client: Client,+ sessionId: string,+ options: { afterSeq: number; limit: number; apply: boolean },+): Promise<{ busy: true } | { busy: false; scanned: number; changed: number; afterSeq: number }> {+ if (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 1000)+ throw new Error("Invalid page limit");+ await client.query(options.apply ? "BEGIN" : "BEGIN READ ONLY");+ try {+ const locked = await client.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS held", [sessionId]);+ const lease = await client.query("SELECT 1 FROM session_leases WHERE session_id=$1 AND expires_at>$2", [+ sessionId,+ Date.now(),+ ]);+ if (!locked.rows[0]?.held || lease.rows.length) {+ await client.query("ROLLBACK");+ return { busy: true };+ }+ const source = (+ await client.query("SELECT * FROM session_entries WHERE session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3", [+ sessionId,+ options.afterSeq,+ options.limit,+ ])+ ).rows.map(rowToEntry);+ if (source.some((entry, index) => entry.seq !== options.afterSeq + index + 1))+ throw new Error("Legacy history has a sequence gap");+ if (source.length < options.limit) {+ const tail = source.at(-1)?.seq ?? options.afterSeq;+ const extra = await client.query(+ "SELECT 1 FROM session_transcript_entries WHERE session_id=$1 AND (seq>$2 OR seq IS NULL OR seq<0) LIMIT 1",+ [sessionId, tail],+ );+ if (extra.rows.length) throw new Error("Canonical history has an extra entry");+ }+ if (!source.length) {+ await client.query("COMMIT");+ return { busy: false, scanned: 0, changed: 0, afterSeq: options.afterSeq };+ }+ const afterSeq = source.at(-1)!.seq;+ const readCanonical = async () =>+ (+ await client.query(+ "SELECT * FROM session_transcript_entries WHERE session_id=$1 AND seq>$2 AND seq<=$3 ORDER BY seq",+ [sessionId, options.afterSeq, afterSeq],+ )+ ).rows.map(rowToEntry);+ const before = await readCanonical();+ const bySeq = new Map(before.map((entry) => [entry.seq, entry]));+ const sourceSeqs = new Set(source.map((entry) => entry.seq));+ if (before.some((entry) => !sourceSeqs.has(entry.seq))) throw new Error("Canonical history has an extra entry");+ const missing = source.filter((entry) => canonicalJson(entry) !== canonicalJson(bySeq.get(entry.seq)));+ if (options.apply && missing.length) {+ const next = (+ await client.query("SELECT COALESCE(MAX(seq),-1)+1 AS seq FROM session_tape WHERE session_id=$1", [sessionId])+ ).rows[0].seq;+ const records = missing.map((entry, ordinal) => ({+ ordinal,+ payload: jsonbSafeStringify(tapeTranscriptEntryRecord(entry).payload),+ scope_label: entry.scopeLabel,+ entry_seq: entry.seq,+ }));+ await client.query(+ `INSERT INTO session_tape(session_id,seq,kind,payload,scope_label,entry_seq,created_at)+ SELECT $1,$2::int+r.ordinal,'annotation',r.payload,r.scope_label,r.entry_seq,$4+ FROM jsonb_to_recordset($3::jsonb) AS r(ordinal int,payload text,scope_label text,entry_seq int)`,+ [sessionId, Number(next), jsonbSafeStringify(records), Date.now()],+ );+ if (canonicalJson(await readCanonical()) !== canonicalJson(source))+ throw new Error("Canonical transcript verification failed");+ }+ await client.query("COMMIT");+ return { busy: false, scanned: source.length, changed: missing.length, afterSeq };+ } catch (error) {+ await client.query("ROLLBACK");+ throw error;+ }+}diff --git a/scripts/migrate-transcript-tape.ts b/scripts/migrate-transcript-tape.tsnew file mode 100644index 000000000..12d536951--- /dev/null+++ b/scripts/migrate-transcript-tape.ts@@ -0,0 +1,75 @@+import { parseArgs } from "node:util";+import pg from "pg";+import { migrateTranscriptPage } from "./lib/transcript-tape-migration.ts";++const { values } = parseArgs({+ options: {+ apply: { type: "boolean", default: false },+ after: { type: "string", default: "" },+ "through-created-at": { type: "string", default: String(Date.now()) },+ "page-size": { type: "string", default: "250" },+ "max-sessions": { type: "string" },+ },+});+const pageSize = Number(values["page-size"]);+const through = Number(values["through-created-at"]);+const maxSessions = values["max-sessions"] === undefined ? Infinity : Number(values["max-sessions"]);+if (+ !Number.isInteger(pageSize) ||+ pageSize < 1 ||+ pageSize > 1000 ||+ !Number.isSafeInteger(through) ||+ through < 0 ||+ (maxSessions !== Infinity && (!Number.isInteger(maxSessions) || maxSessions < 1))+)+ throw new Error("Invalid migration bounds");+if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required");+const client = new pg.Client({+ connectionString: process.env.DATABASE_URL,+ connectionTimeoutMillis: 10_000,+ application_name: "qm-transcript-tape-migration",+ options: "-c statement_timeout=30000 -c lock_timeout=1000",+});+await client.connect();+let cursor = values.after!;+const summary = { sessions: 0, entries: 0, changed: 0, busy: 0 };+const report = (event: string) =>+ console.log(JSON.stringify({ event, apply: values.apply, cursor, throughCreatedAt: through, ...summary }));+report("start");+try {+ for (;;) {+ const batch = (+ await client.query("SELECT id FROM sessions WHERE id>$1 AND created_at<=$2 ORDER BY id LIMIT $3", [+ cursor,+ through,+ Math.min(100, maxSessions - summary.sessions),+ ])+ ).rows;+ if (!batch.length) break;+ for (const { id } of batch) {+ let afterSeq = -1;+ for (;;) {+ const page = await migrateTranscriptPage(client, id, { afterSeq, limit: pageSize, apply: values.apply! });+ if (page.busy) {+ summary.busy++;+ console.log(JSON.stringify({ event: "busy", sessionId: id, afterSeq }));+ break;+ }+ summary.entries += page.scanned;+ summary.changed += page.changed;+ afterSeq = page.afterSeq;+ if (page.scanned < pageSize) break;+ }+ summary.sessions++;+ cursor = id;+ }+ report("progress");+ if (summary.sessions >= maxSessions) break;+ }+ const partial = summary.sessions >= maxSessions || values.after !== "";+ report(partial ? "partial" : "complete");+ process.exitCode = partial ? 2 : 0;+ if (summary.busy || (!values.apply && summary.changed)) process.exitCode = 1;+} finally {+ await client.end();+}diff --git a/src/harness/tape-projection.ts b/src/harness/tape-projection.tsindex 17ead3efb..0a76f3b55 100644--- a/src/harness/tape-projection.ts+++ b/src/harness/tape-projection.ts@@ -8,7 +8,6 @@ import { assistantDroppedAtReplay } from "./tape-fold.ts";import { textFromContent, thinkingBlocksFromContent } from "./pi-harness.ts";import { swallow } from "../util/errors.ts";-const UNSERVABLE_MEMO_CAP = 10_000;const SUFFIX_ROW_CAP_MIN = 100;const SUFFIX_ROW_CAP_MAX = 5_000;@@ -202,7 +201,11 @@ export function projectTapeEntries(tapeRows: readonly TapeRecord[],opts?: { anchored?: boolean },): TapeProjection | null {- const sliced = renderableTapeSlice(tapeRows);+ const sliced = renderableTapeSlice(+ tapeRows.filter(+ (row) => row.kind !== "annotation" || (row.payload as { event?: unknown } | null)?.event !== "transcript_entry",+ ),+ );let rows = sliced;let base = -1;if (opts?.anchored) {@@ -394,7 +397,8 @@ export function projectTapeEntries(type TranscriptStore = Pick<SessionStore,"getEntries" | "visibleEntries" | "getTape" | "latestEntrySeq" | "participantWindowsOf"->;+> &+ Partial<Pick<SessionStore, "getTranscriptEntries">>;interface TranscriptRead {entries: SessionEntry[];@@ -406,19 +410,6 @@ export interface TranscriptSource {forViewer(sessionId: string, principalId: string, opts?: { limit?: number }): Promise<TranscriptRead>;}-function createUnservableMemo() {- const ids = new Set<string>();- return {- has: (sessionId: string): boolean => ids.has(sessionId),- remember(sessionId: string): void {- if (ids.size >= UNSERVABLE_MEMO_CAP) ids.clear();- ids.add(sessionId);- },- };-}--const unservableTapes = createUnservableMemo();-interface ProjectedRead {entries: SessionEntry[];anchored: boolean;@@ -427,10 +418,16 @@ interface ProjectedRead {export function createTranscriptSource(sessions: TranscriptStore): TranscriptSource {const projected = async (sessionId: string, limit?: number): Promise<ProjectedRead | null> => {- if (unservableTapes.has(sessionId)) return null;try {const latest = await sessions.latestEntrySeq(sessionId);if (latest < 0) return { entries: [], anchored: false, base: -1 };+ if (sessions.getTranscriptEntries) {+ const entries = await sessions.getTranscriptEntries(sessionId, limit === undefined ? undefined : { limit });+ const first = limit === undefined ? 0 : Math.max(0, latest - limit + 1);+ if (entries.length === latest - first + 1 && entries.every((entry, i) => entry.seq === first + i)) {+ return { entries, anchored: first > 0, base: first - 1 };+ }+ }let rows: TapeRecord[];let anchored = false;if (limit !== undefined) {@@ -441,7 +438,6 @@ export function createTranscriptSource(sessions: TranscriptStore): TranscriptSourows = await sessions.getTape(sessionId);}if (!anchored && tapeHasRenderBlockers(rows)) {- unservableTapes.remember(sessionId);return null;}const projection = projectTapeEntries(sessionId, rows, { anchored });diff --git a/src/sessions/memory-session-store.ts b/src/sessions/memory-session-store.tsindex 3966d6303..1ad67d560 100644--- a/src/sessions/memory-session-store.ts+++ b/src/sessions/memory-session-store.ts@@ -42,6 +42,8 @@ import {sessionCategory,sessionOrigin,userMessagePreview,+ tapeTranscriptEntryRecord,+ transcriptEntryFromTape,} from "./session-store.ts";import { SECURITY_SCREEN_STEP, screenPayloadFromEnvelope } from "../security/security-posture.ts";@@ -247,7 +249,16 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStorescopeLabel: entry.scopeLabel as ScopeId,createdAt: now(),};+ const mirrored = structuredClone(tapeTranscriptEntryRecord(full));log.push(full);+ const tapeLog = tape.get(lease.sessionId) ?? [];+ tapeLog.push({+ ...mirrored,+ sessionId: lease.sessionId,+ seq: tapeLog.length,+ createdAt: now(),+ });+ tape.set(lease.sessionId, tapeLog);const text = SEARCHABLE_ENTRY_TYPES.has(full.type) ? entrySearchText(full.payload) : null;if (text?.trim()) {const index = searchIndex.get(full.sessionId) ?? [];@@ -265,6 +276,19 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStorereturn opts?.limit !== undefined ? filtered.slice(-opts.limit) : filtered;},+ async getTranscriptEntries(sessionId, opts?: GetEntriesOptions) {+ const projected = new Map<number, SessionEntry>();+ for (const row of tape.get(sessionId) ?? []) {+ const entry = transcriptEntryFromTape(row);+ if (entry) projected.set(entry.seq, entry);+ }+ const filtered = [...projected.values()]+ .filter((entry) => entry.seq >= (opts?.sinceSeq ?? 0))+ .sort((a, b) => a.seq - b.seq);+ if (opts?.limit === 0) return [];+ return opts?.limit === undefined ? filtered : filtered.slice(-opts.limit);+ },+async getContextWindow(sessionId) {return contextWindowFromEntries(entries.get(sessionId) ?? []);},@@ -284,7 +308,16 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStoreif (!entry.payload || typeof entry.payload !== "object") continue;const payload = { ...(entry.payload as Record<string, unknown>) };delete payload.securityTainted;+ if (!("securityTainted" in (entry.payload as object))) continue;entry.payload = payload;+ const tapeLog = tape.get(sessionId) ?? [];+ tapeLog.push({+ ...structuredClone(tapeTranscriptEntryRecord(entry)),+ sessionId,+ seq: tapeLog.length,+ createdAt: now(),+ });+ tape.set(sessionId, tapeLog);}return true;},diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex f5149a763..cbf6b571a 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -42,6 +42,7 @@ import {stableOriginPattern,threadRefCronIdExpr,userMessagePreview,+ tapeTranscriptEntryRecord,} from "./session-store.ts";import { SECURITY_SCREEN_STEP, screenPayloadFromEnvelope } from "../security/security-posture.ts";@@ -156,7 +157,7 @@ function rowToParticipantWindow(r: Record<string, unknown>): ParticipantWindow {};}-function rowToEntry(r: Record<string, unknown>): SessionEntry {+export function rowToEntry(r: Record<string, unknown>): SessionEntry {return {sessionId: r.session_id as string,seq: Number(r.seq),@@ -534,6 +535,25 @@ export function createPostgresSessionStore(connectionString: string, opts: Store`ALTER TABLE session_tape ADD COLUMN IF NOT EXISTS source_role TEXT`,],},+ {+ id: "sessions/store/0017-transcript-entries",+ statements: [+ `CREATE INDEX IF NOT EXISTS session_tape_transcript_entries+ ON session_tape(session_id, entry_seq DESC, seq DESC)+ WHERE kind = 'annotation' AND safe_json(payload)->>'event' = 'transcript_entry'`,+ `CREATE OR REPLACE VIEW session_transcript_entries AS+ SELECT DISTINCT ON (session_id, entry_seq)+ session_id, entry_seq AS seq,+ (safe_json(payload)->'entry'->>'parentSeq')::int AS parent_seq,+ safe_json(payload)->'entry'->>'type' AS type,+ (safe_json(payload)->'entry'->'payload')::text AS payload,+ scope_label,+ (safe_json(payload)->'entry'->>'at')::bigint AS created_at+ FROM session_tape t+ WHERE kind = 'annotation' AND safe_json(payload)->>'event' = 'transcript_entry'+ ORDER BY t.session_id, t.entry_seq DESC, t.seq DESC`,+ ],+ },],[{@@ -763,6 +783,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Store"INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7)",[full.sessionId, full.seq, full.parentSeq, full.type, stored, full.scopeLabel, full.createdAt],);+ await insertTapeRow(client, full.sessionId, tapeTranscriptEntryRecord(full));await client.query(`UPDATE sessionsSET last_activity = CASE WHEN last_activity >= $2::bigint - ${LAST_ACTIVITY_DEBOUNCE_MS}@@ -786,13 +807,28 @@ export function createPostgresSessionStore(connectionString: string, opts: Store},async clearSecurityTaint(sessionId) {- const updated = await q(- "UPDATE session_entries SET payload = (payload::jsonb - 'securityTainted')::text " +- "WHERE session_id = $1 AND payload LIKE '%\"securityTainted\"%' RETURNING 1",- [sessionId],+ return withPgTransaction(await pool(), async (client) => {+ await lockSession(client, sessionId);+ const updated = await client.query(+ "UPDATE session_entries SET payload = (payload::jsonb - 'securityTainted')::text " ++ "WHERE session_id = $1 AND jsonb_typeof(payload::jsonb) = 'object' AND payload::jsonb ? 'securityTainted' RETURNING *",+ [sessionId],+ );+ for (const row of updated.rows) {+ await insertTapeRow(client, sessionId, tapeTranscriptEntryRecord(rowToEntry(row)));+ }+ if (updated.rows.length > 0) return true;+ return (await client.query("SELECT 1 FROM sessions WHERE id = $1", [sessionId])).rows.length === 1;+ });+ },++ async getTranscriptEntries(sessionId, opts?: GetEntriesOptions) {+ const rows = await q(+ "SELECT * FROM session_transcript_entries WHERE session_id = $1 AND seq >= $2 ORDER BY seq DESC" ++ (opts?.limit === undefined ? "" : " LIMIT $3"),+ [sessionId, opts?.sinceSeq ?? 0, ...(opts?.limit === undefined ? [] : [opts.limit])],);- if (updated.length > 0) return true;- return (await q("SELECT 1 FROM sessions WHERE id = $1", [sessionId])).length === 1;+ return rows.map(rowToEntry).reverse();},async appendTape(lease, rec: NewTapeRecord): Promise<TapeRecord> {diff --git a/src/sessions/session-store.ts b/src/sessions/session-store.tsindex 8109ed653..2e4c3d03d 100644--- a/src/sessions/session-store.ts+++ b/src/sessions/session-store.ts@@ -180,6 +180,44 @@ export function tapeEntryMirrorRecord(entry: {};}+export function tapeTranscriptEntryRecord(entry: SessionEntry): NewTapeRecord {+ return {+ kind: "annotation",+ payload: {+ event: "transcript_entry",+ entry: { type: entry.type, payload: entry.payload, at: entry.createdAt, parentSeq: entry.parentSeq },+ },+ scopeLabel: entry.scopeLabel,+ entrySeq: entry.seq,+ };+}++export function transcriptEntryFromTape(row: TapeRecord): SessionEntry | null {+ if (row.kind !== "annotation" || row.entrySeq === undefined) return null;+ const payload = row.payload as {+ event?: unknown;+ entry?: { type?: unknown; payload?: unknown; at?: unknown; parentSeq?: unknown };+ } | null;+ const entry = payload?.entry;+ if (+ payload?.event !== "transcript_entry" ||+ !entry ||+ typeof entry.type !== "string" ||+ typeof entry.at !== "number" ||+ (entry.parentSeq !== null && typeof entry.parentSeq !== "number")+ )+ return null;+ return {+ sessionId: row.sessionId,+ seq: row.entrySeq,+ parentSeq: entry.parentSeq,+ type: entry.type as EntryType,+ payload: entry.payload ?? null,+ scopeLabel: row.scopeLabel,+ createdAt: entry.at,+ };+}+export type TranscriptAppendSessions = Pick<SessionStore, "append" | "appendTape" | "latestEntrySeq" | "tapeCoverage">;export async function appendEntryOutsideTurn(@@ -671,6 +709,7 @@ export interface SessionStore {append(lease: Lease, entry: NewEntry): Promise<SessionEntry>;getEntries(sessionId: string, opts?: GetEntriesOptions): Promise<SessionEntry[]>;+ getTranscriptEntries(sessionId: string, opts?: GetEntriesOptions): Promise<SessionEntry[]>;getContextWindow(sessionId: string): Promise<ContextWindow>;getEntry(sessionId: string, seq: number): Promise<SessionEntry | undefined>;latestEntrySeq(sessionId: string): Promise<number>;diff --git a/test/message-revisions.test.ts b/test/message-revisions.test.tsindex 2760b2e08..ead3a465f 100644--- a/test/message-revisions.test.ts+++ b/test/message-revisions.test.ts@@ -201,7 +201,7 @@ test("a channel thread reply's edit reaches the thread session via the sub root"assert.equal(marks[0]!.name, "Josh");});-test("when the tape is not contiguous the marker entry still lands but the tape stays untouched", async () => {+test("when model tape is not contiguous the revision is mirrored without advancing model coverage", async () => {const sessions = createMemorySessionStore();const session = await sessions.getOrCreateByThread(`dm:${DM}`, "dm", SCOPE, undefined, "slack");const { lease } = await sessions.acquireLease(session.id, "turn");@@ -211,7 +211,13 @@ test("when the tape is not contiguous the marker entry still lands but the tapeawait recordMessageRevisions(sessions, [edit()]);assert.equal(revisions(await sessions.getEntries(session.id)).length, 1);- assert.equal((await sessions.getTape(session.id)).length, 0);+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), await sessions.getEntries(session.id));+ assert.equal(await sessions.tapeCoverage(session.id), -1);+ assert.ok(+ (await sessions.getTape(session.id)).every(+ (row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry",+ ),+ );});test("the marker renders into model context on the entries-replay path", () => {diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.tsindex dfcc40392..059173c94 100644--- a/test/postgres-store.test.ts+++ b/test/postgres-store.test.ts@@ -1,4 +1,5 @@import "./run-availability.ts";+import { migrateTranscriptPage } from "../scripts/lib/transcript-tape-migration.ts";import { test, before } from "node:test";import { randomUUID } from "node:crypto";import assert from "node:assert/strict";@@ -380,8 +381,9 @@ test("pg latestEntrySeq, participant windows, and tape meta attachments round-trmeta: { bareText: "one", attachments, sourceRole: "agent" },});const rows = await s.getTape(session.id);- assert.deepEqual(rows[0]!.meta?.attachments, attachments);- assert.equal(rows[0]!.meta?.sourceRole, "agent");+ const message = rows.find((row) => row.kind === "message")!;+ assert.deepEqual(message.meta?.attachments, attachments);+ assert.equal(message.meta?.sourceRole, "agent");});test("pg participant tenure remains exact when every event shares a timestamp", { skip }, async () => {@@ -2234,3 +2236,197 @@ test("pg session store: bounded participant listing is recent and optional", { s);assert.deepEqual(await store.listByParticipant("bounded-user", { limit: 0 }), []);});++test(+ "pg canonical transcript annotations are exact and taint release preserves original identity",+ { skip },+ async () => {+ const s = createPostgresSessionStore(URL!);+ const scope = scopeId("personal", "canonical-tape");+ const session = await s.getOrCreateByThread("pg-canonical-tape", "dm", scope);+ const { lease } = await s.acquireLease(session.id);+ assert.ok(lease);+ for (let i = 0; i < 4; i++)+ await s.append(lease, {+ type: "tool_result",+ scopeLabel: scope,+ payload: { tool: "execute", callId: `c${i}`, result: "denied", isError: true, code: 1, securityTainted: true },+ });+ const before = await s.getEntries(session.id);+ assert.deepEqual(await s.getTranscriptEntries(session.id), before);+ assert.deepEqual(await s.getTranscriptEntries(session.id, { limit: 2 }), before.slice(-2));+ assert.deepEqual(await s.getTranscriptEntries(session.id, { sinceSeq: 1, limit: 2 }), before.slice(-2));+ assert.equal(await s.tapeCoverage(session.id), -1);+ assert.equal(await s.clearSecurityTaint(session.id), true);+ const after = await s.getEntries(session.id);+ assert.deepEqual(await s.getTranscriptEntries(session.id), after);+ assert.equal(after[0]!.createdAt, before[0]!.createdAt);+ assert.equal(after[0]!.parentSeq, before[0]!.parentSeq);+ assert.equal((after[0]!.payload as { securityTainted?: boolean }).securityTainted, undefined);+ assert.equal((await s.getTape(session.id)).length, 8);+ assert.equal(await s.clearSecurityTaint(session.id), true);+ assert.equal((await s.getTape(session.id)).length, 8);+ assert.equal(await s.tapeCoverage(session.id), -1);+ await s.releaseLease(lease);+ },+);++test("pg transcript write failure rolls back the entry and its counters", { skip }, async () => {+ const s = createPostgresSessionStore(URL!);+ const scope = scopeId("personal", "canonical-failure");+ const session = await s.getOrCreateByThread("pg-canonical-failure", "dm", scope);+ const { lease } = await s.acquireLease(session.id);+ assert.ok(lease);+ const pg = (await import("pg")).default;+ const client = new pg.Client({ connectionString: URL! });+ await client.connect();+ await client.query(+ `CREATE FUNCTION reject_transcript_test() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'transcript unavailable'; END $$`,+ );+ await client.query(+ `CREATE TRIGGER reject_transcript_test BEFORE INSERT ON session_tape FOR EACH ROW WHEN (NEW.session_id = '${session.id}') EXECUTE FUNCTION reject_transcript_test()`,+ );+ try {+ await assert.rejects(+ s.append(lease, { type: "user", payload: { text: "must rollback" }, scopeLabel: scope }),+ /transcript unavailable/,+ );+ assert.deepEqual(await s.getEntries(session.id), []);+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);+ assert.equal(await s.latestEntrySeq(session.id), -1);+ const counters = (await client.query("SELECT messages FROM sessions WHERE id=$1", [session.id])).rows[0];+ assert.equal(counters.messages, 0);+ } finally {+ await client.query("DROP TRIGGER reject_transcript_test ON session_tape");+ await client.query("DROP FUNCTION reject_transcript_test()");+ await client.end();+ await s.releaseLease(lease);+ }+});++test("pg transcript backfill is bounded, idempotent, and independent of model coverage", { skip }, async () => {+ const s = createPostgresSessionStore(URL!);+ const scope = scopeId("personal", "canonical-backfill");+ const session = await s.getOrCreateByThread("pg-canonical-backfill", "dm", scope);+ const pg = (await import("pg")).default;+ const client = new pg.Client({ connectionString: URL! });+ await client.connect();+ try {+ await client.query(+ `INSERT INTO session_entries(session_id,seq,parent_seq,type,payload,scope_label,created_at)+ SELECT $1,i,NULLIF(i-1,-1),'user',json_build_object('text','history '||i,'securityTainted',true)::text,$2,123000+i+ FROM generate_series(0,619) i`,+ [session.id, scope],+ );+ const dry = await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 200, apply: false });+ assert.deepEqual(dry, { busy: false, scanned: 200, changed: 200, afterSeq: 199 });+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);+ for (let afterSeq = -1; afterSeq < 619;) {+ const page = await migrateTranscriptPage(client, session.id, { afterSeq, limit: 200, apply: true });+ assert.ok(!page.busy);+ assert.ok(page.scanned <= 200);+ afterSeq = page.afterSeq;+ }+ assert.deepEqual(await s.getTranscriptEntries(session.id), await s.getEntries(session.id));+ assert.equal(await s.tapeCoverage(session.id), -1);+ const repeated = await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 200, apply: true });+ assert.deepEqual(repeated, { busy: false, scanned: 200, changed: 0, afterSeq: 199 });+ assert.equal((await s.getTape(session.id)).length, 620);+ await s.clearSecurityTaint(session.id);+ assert.deepEqual(await s.getTranscriptEntries(session.id), await s.getEntries(session.id));+ } finally {+ await client.end();+ }+});++test("pg transcript backfill refuses an active lease or writer transaction", { skip }, async () => {+ const s = createPostgresSessionStore(URL!);+ const scope = scopeId("personal", "canonical-busy");+ const session = await s.getOrCreateByThread("pg-canonical-busy", "dm", scope);+ const pg = (await import("pg")).default;+ const client = new pg.Client({ connectionString: URL! });+ const writer = new pg.Client({ connectionString: URL! });+ await client.connect();+ await writer.connect();+ try {+ const { lease } = await s.acquireLease(session.id);+ assert.ok(lease);+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {+ busy: true,+ });+ await s.releaseLease(lease);+ await writer.query("BEGIN");+ await writer.query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))", [session.id]);+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {+ busy: true,+ });+ await writer.query("ROLLBACK");+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {+ busy: false,+ scanned: 0,+ changed: 0,+ afterSeq: -1,+ });+ } finally {+ await writer.end();+ await client.end();+ }+});++test(+ "pg transcript migration rejects extra canonical tails and entries in an empty legacy session",+ { skip },+ async () => {+ const s = createPostgresSessionStore(URL!);+ const pg = (await import("pg")).default;+ const client = new pg.Client({ connectionString: URL! });+ await client.connect();+ try {+ for (const originalCount of [0, 3]) {+ const scope = scopeId("personal", `extra-canonical-${originalCount}`);+ const session = await s.getOrCreateByThread(`pg-extra-canonical-${originalCount}`, "dm", scope);+ const { lease } = await s.acquireLease(session.id);+ assert.ok(lease);+ for (let i = 0; i <= originalCount; i++)+ await s.append(lease, { type: "user", payload: { text: `row ${i}` }, scopeLabel: scope });+ await s.releaseLease(lease);+ await client.query("DELETE FROM session_entries WHERE session_id=$1 AND seq=$2", [session.id, originalCount]);+ for (const apply of [false, true]) {+ await assert.rejects(+ migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply }),+ /extra entry/,+ );+ }+ assert.equal((await s.getTranscriptEntries(session.id)).length, originalCount + 1);+ }+ } finally {+ await client.end();+ }+ },+);++test("pg transcript migration refuses gapped legacy history", { skip }, async () => {+ const s = createPostgresSessionStore(URL!);+ const pg = (await import("pg")).default;+ const client = new pg.Client({ connectionString: URL! });+ await client.connect();+ try {+ const scope = scopeId("personal", "gapped-transcript");+ const session = await s.getOrCreateByThread("pg-gapped-transcript", "dm", scope);+ const { lease } = await s.acquireLease(session.id);+ assert.ok(lease);+ for (let i = 0; i < 3; i++)+ await s.append(lease, { type: "user", payload: { text: `row ${i}` }, scopeLabel: scope });+ await s.releaseLease(lease);+ await client.query("DELETE FROM session_entries WHERE session_id=$1 AND seq=1", [session.id]);+ await client.query("DELETE FROM session_tape WHERE session_id=$1", [session.id]);+ for (const apply of [false, true])+ await assert.rejects(+ migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply }),+ /sequence gap/,+ );+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);+ } finally {+ await client.end();+ }+});diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.tsindex 2720c24d5..c8252467a 100644--- a/test/run-result-delivery.test.ts+++ b/test/run-result-delivery.test.ts@@ -281,8 +281,14 @@ test("a parked run's failure lands as a turn_failure entry in the run's own sessrunId: "r-1",});const tape = await sessions.getTape(session.id);- assert.equal(tape.length, 1, "a turnEnd checkpoint keeps the tape projection servable");- assert.equal(tape[0]!.kind, "annotation");+ assert.equal(tape.length, 2);+ assert.equal(+ tape.filter((row) => (row.payload as { turnEnd?: boolean }).turnEnd === true).length,+ 1,+ "exactly one turnEnd checkpoint keeps model coverage intact",+ );+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), entries);+ assert.equal(await sessions.tapeCoverage(session.id), entries.at(-1)!.seq);assert.equal(await recordRunFailureEntry(sessions, failedRun()), false, "recording is idempotent");assert.equal((await sessions.getEntries(session.id)).length, 1);diff --git a/test/session-view-bounds.test.ts b/test/session-view-bounds.test.tsindex 517ea137c..0347af096 100644--- a/test/session-view-bounds.test.ts+++ b/test/session-view-bounds.test.ts@@ -52,6 +52,17 @@ test("a tailTurns view is bounded yet reports the same earlierEntries as the ful}});+function withoutCanonicalTranscript(sessions: SessionStore) {+ const getTape = sessions.getTape.bind(sessions);+ sessions.getTranscriptEntries = async () => [];+ sessions.getTape = async (sessionId, opts) => {+ const rows = (await getTape(sessionId)).filter(+ (row) => (row.payload as { event?: string }).event !== "transcript_entry" && row.seq > (opts?.sinceSeq ?? -1),+ );+ return opts?.limit === undefined ? rows : rows.slice(-opts.limit);+ };+}+async function coarseForeignSession(sessions: SessionStore, turns = 1) {const scope = scopeId("personal", "U1");const session = await sessions.getOrCreateByThread("web:U1:coarse-pins", "dm", scope, undefined, "web");@@ -103,6 +114,7 @@ test("bounded earlierEntries on a coarse foreign session counts renderable entriconst built = freshApp();built.runtime.start();try {+ withoutCanonicalTranscript(built.sessions);const { session } = await coarseForeignSession(built.sessions, 30);const full = (await built.app.getSession(session.id))!;assert.equal(full.earlierEntries ?? 0, 0, "the unbounded read reports nothing earlier");@@ -133,6 +145,7 @@ test("pins on entries a coarse projection drops still resolve through the targetconst built = freshApp();built.runtime.start();try {+ withoutCanonicalTranscript(built.sessions);const { session, narration } = await coarseForeignSession(built.sessions);const view = (await built.app.getSessionForViewer(session.id, "U1"))!;assert.deepEqual(@@ -153,3 +166,22 @@ test("pins on entries a coarse projection drops still resolve through the targetawait built.runtime.stop();}});++test("canonical transcripts retain foreign harness tool detail and exact bounded counts", async () => {+ const built = freshApp();+ built.runtime.start();+ try {+ const { session, narration } = await coarseForeignSession(built.sessions, 30);+ const full = (await built.app.getSessionForViewer(session.id, "U1", { tailTurns: 9999 }))!;+ assert.deepEqual(full.entries, await built.sessions.getEntries(session.id));+ assert.equal(full.entries.length, 150);+ const bounded = (await built.app.getSessionForViewer(session.id, "U1", { tailTurns: 1 }))!;+ assert.deepEqual(bounded.entries, full.entries.slice(-5));+ assert.equal(bounded.earlierEntries, 145);+ const pinned = await built.app.pinConversationItem("web:U1:coarse-pins", "U1", { entrySeq: narration.seq });+ assert.ok("pin" in pinned && pinned.pin);+ assert.equal(pinned.pin.preview, "queue narration detail");+ } finally {+ await built.runtime.stop();+ }+});diff --git a/test/tape-canonical-transcript.test.ts b/test/tape-canonical-transcript.test.tsnew file mode 100644index 000000000..456f6ba05--- /dev/null+++ b/test/tape-canonical-transcript.test.ts@@ -0,0 +1,119 @@+import assert from "node:assert/strict";+import { test } from "node:test";+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";+import { createTranscriptSource } from "../src/harness/tape-projection.ts";+import { foldTape } from "../src/harness/tape-fold.ts";+import { tapeTranscriptEntryRecord } from "../src/sessions/session-store.ts";+import type { ScopeId } from "../src/types.ts";++const scope = "personal:[email protected]" as ScopeId;++async function scenario() {+ const store = createMemorySessionStore();+ const session = await store.getOrCreateByThread("canonical-transcript", "dm", scope);+ const { lease } = await store.acquireLease(session.id);+ assert.ok(lease);+ return { store, session, lease };+}++test("exact tape annotations retain tool fields without advancing model coverage", async () => {+ const { store, session, lease } = await scenario();+ const entry = await store.append(lease, {+ type: "tool_result",+ scopeLabel: scope,+ payload: {+ tool: "execute",+ callId: "failed",+ stdout: "",+ stderr: "denied",+ code: 1,+ isError: true,+ result: "denied",+ },+ });+ assert.deepEqual(await store.getTranscriptEntries(session.id), [entry]);+ assert.equal(await store.tapeCoverage(session.id), -1);+ assert.deepEqual(foldTape(await store.getTape(session.id)), []);+ const source = createTranscriptSource({+ ...store,+ getEntries: async () => {+ throw new Error("legacy read");+ },+ });+ assert.deepEqual((await source.forRender(session.id)).entries, [entry]);+});++test("taint release supersedes the exact annotation without changing identity or native model history", async () => {+ const { store, session, lease } = await scenario();+ await store.append(lease, { type: "user", payload: { text: "reviewed", securityTainted: true }, scopeLabel: scope });+ const before = await store.getTranscriptEntries(session.id);+ const beforeRows = await store.getTape(session.id);+ assert.equal(await store.clearSecurityTaint(session.id), true);+ assert.deepEqual(await store.getTranscriptEntries(session.id), await store.getEntries(session.id));+ assert.deepEqual(await store.getTranscriptEntries(session.id), [{ ...before[0]!, payload: { text: "reviewed" } }]);+ assert.equal((await store.getTape(session.id)).length, beforeRows.length + 1);+ assert.equal(await store.tapeCoverage(session.id), -1);+ assert.deepEqual(foldTape(await store.getTape(session.id)), []);+ assert.equal(await store.clearSecurityTaint(session.id), true);+ assert.equal((await store.getTape(session.id)).length, beforeRows.length + 1);+});++test("canonical transcript reads preserve full histories, bounded tails and participant windows", async () => {+ const { store, session, lease } = await scenario();+ for (let i = 0; i < 620; i++) {+ if (i === 300) await store.addParticipant(session.id, "viewer");+ if (i === 550) await store.removeParticipant(session.id, "viewer");+ await store.append(lease, { type: "user", payload: { text: `message ${i}` }, scopeLabel: scope });+ }+ const entries = await store.getEntries(session.id);+ const visible = await store.visibleEntries(session.id, "viewer");+ const calls: Array<number | undefined> = [];+ const source = createTranscriptSource({+ ...store,+ getEntries: async () => {+ throw new Error("legacy read");+ },+ visibleEntries: async () => {+ throw new Error("legacy viewer read");+ },+ getTape: async () => {+ throw new Error("native tape read");+ },+ getTranscriptEntries: (id, opts) => {+ calls.push(opts?.limit);+ return store.getTranscriptEntries(id, opts);+ },+ });+ assert.deepEqual((await source.forRender(session.id)).entries, entries);+ assert.deepEqual(await source.forRender(session.id, { limit: 6 }), { entries: entries.slice(-6), earlier: 614 });+ assert.equal(calls.at(-1), 6);+ assert.deepEqual((await source.forRender(session.id, { sinceSeq: 615 })).entries, entries.slice(615));+ assert.deepEqual((await source.forViewer(session.id, "viewer")).entries, visible);+ assert.deepEqual((await source.forViewer(session.id, "stranger")).entries, []);+});++test("partial canonical coverage uses legacy history until every sequence is present", async () => {+ const { store, session, lease } = await scenario();+ for (let i = 0; i < 3; i++) await store.append(lease, { type: "user", payload: { text: `${i}` }, scopeLabel: scope });+ const entries = await store.getEntries(session.id);+ let reads = 0;+ const source = createTranscriptSource({+ ...store,+ getTranscriptEntries: async () => entries.slice(1),+ getTape: async () => [],+ getEntries: async () => {+ reads++;+ return entries;+ },+ });+ assert.deepEqual((await source.forRender(session.id)).entries, entries);+ assert.equal(reads, 1);+});++test("historical canonical annotations preserve original parent, scope and timestamp", async () => {+ const { store, session, lease } = await scenario();+ const entry = await store.append(lease, { type: "user", payload: { text: "original" }, scopeLabel: scope });+ const historical = { ...entry, parentSeq: null, createdAt: 123, scopeLabel: "org:historic" as ScopeId };+ await store.appendTape(lease, tapeTranscriptEntryRecord(historical));+ assert.deepEqual(await store.getTranscriptEntries(session.id), [historical]);+});diff --git a/test/tape-projection.test.ts b/test/tape-projection.test.tsindex 93a1a990c..62f92f68e 100644--- a/test/tape-projection.test.ts+++ b/test/tape-projection.test.ts@@ -22,7 +22,7 @@ interface Sim {}async function simSession(threadRef = "dm:projection-test"): Promise<Sim> {- const store = createMemorySessionStore({ now: () => CLOCK });+ const store = { ...createMemorySessionStore({ now: () => CLOCK }), getTranscriptEntries: async () => [] };const session = await store.getOrCreateByThread(threadRef, "dm", scope);const { lease } = await store.acquireLease(session.id);assert.ok(lease);diff --git a/test/tape-retirement.test.ts b/test/tape-retirement.test.tsindex e83f611a9..fa3c67199 100644--- a/test/tape-retirement.test.ts+++ b/test/tape-retirement.test.ts@@ -395,7 +395,11 @@ test("a tainted uncovered session is refused without a coverage claim", async ()const entries = await sim.store.getEntries(sim.session.id);const outcome = await appendRenderImport(sim.store, sim.lease, entries, scope, true);assert.equal(outcome, "unservable-fold");- assert.equal((await sim.store.getTape(sim.session.id)).length, 0);+ assert.ok(+ (await sim.store.getTape(sim.session.id)).every(+ (row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry",+ ),+ );assert.equal(await sim.store.tapeCoverage(sim.session.id), -1);});diff --git a/test/web-transcript-delivery.test.ts b/test/web-transcript-delivery.test.tsindex 454cd755c..6db325d17 100644--- a/test/web-transcript-delivery.test.ts+++ b/test/web-transcript-delivery.test.ts@@ -116,7 +116,11 @@ test("a session whose tape is already behind gets the entry but no orphan tape rconst entries = await sessions.getEntries(session.id);assert.equal(entries.length, 2, "the transcript entry still lands");const tape = await sessions.getTape(session.id);- assert.equal(tape.length, 0, "no tape rows without their covering annotation (heal owns this session)");+ assert.ok(+ tape.every((row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry"),+ );+ assert.equal(await sessions.tapeCoverage(session.id), -1, "transcript annotations do not advance model coverage");+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), entries);});test("repeated drains and a restarted decorator never duplicate the transcript entry", async () => {