Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Preserve exact transcripts on the session tape (#1272)

Regan Bell45be37eeae6f
diff --git a/deploy/core/Dockerfile b/deploy/core/Dockerfile
index b811dc97d..f7d154836 100644
--- a/deploy/core/Dockerfile
+++ b/deploy/core/Dockerfile
@@ -17,6 +17,8 @@ RUN npm ci --omit=dev && npm audit --omit=dev --audit-level=moderate \
&& rm -rf /root/.npm /tmp/node-compile-cache
COPY src ./src
+COPY scripts/migrate-transcript-tape.ts ./scripts/migrate-transcript-tape.ts
+COPY scripts/lib/transcript-tape-migration.ts ./scripts/lib/transcript-tape-migration.ts
COPY cli/templates/slack-manifest.json ./cli/templates/slack-manifest.json
COPY skills-seed ./skills-seed
COPY plugins/onboarding ./plugins/onboarding
diff --git a/package.json b/package.json
index 39ec066a5..fd1adff2b 100644
--- a/package.json
+++ b/package.json
@@ -50,7 +50,8 @@
"lint": "eslint .",
"lint:ox": "oxlint --deny-warnings src plugins scripts cli test",
"dev-instance:doctor": "bash scripts/dev-instance.sh doctor",
- "lint:knip": "knip"
+ "lint:knip": "knip",
+ "migrate:transcript-tape": "node scripts/migrate-transcript-tape.ts"
},
"lint-staged": {
"*": "prettier --write --ignore-unknown"
diff --git a/scripts/lib/transcript-tape-migration.ts b/scripts/lib/transcript-tape-migration.ts
new file mode 100644
index 000000000..eb51d9b49
--- /dev/null
+++ b/scripts/lib/transcript-tape-migration.ts
@@ -0,0 +1,84 @@
+import type { Client } from "pg";
+import { rowToEntry } from "../../src/sessions/postgres-session-store.ts";
+import { tapeTranscriptEntryRecord } from "../../src/sessions/session-store.ts";
+import { canonicalJson } from "../../src/util/objects.ts";
+import { jsonbSafeStringify } from "../../src/util/text.ts";
+
+export async function migrateTranscriptPage(
+ client: Client,
+ sessionId: string,
+ options: { afterSeq: number; limit: number; apply: boolean },
+): Promise<{ busy: true } | { busy: false; scanned: number; changed: number; afterSeq: number }> {
+ if (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 1000)
+ throw new Error("Invalid page limit");
+ await client.query(options.apply ? "BEGIN" : "BEGIN READ ONLY");
+ try {
+ const locked = await client.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0)) AS held", [sessionId]);
+ const lease = await client.query("SELECT 1 FROM session_leases WHERE session_id=$1 AND expires_at>$2", [
+ sessionId,
+ Date.now(),
+ ]);
+ if (!locked.rows[0]?.held || lease.rows.length) {
+ await client.query("ROLLBACK");
+ return { busy: true };
+ }
+ const source = (
+ await client.query("SELECT * FROM session_entries WHERE session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3", [
+ sessionId,
+ options.afterSeq,
+ options.limit,
+ ])
+ ).rows.map(rowToEntry);
+ if (source.some((entry, index) => entry.seq !== options.afterSeq + index + 1))
+ throw new Error("Legacy history has a sequence gap");
+ if (source.length < options.limit) {
+ const tail = source.at(-1)?.seq ?? options.afterSeq;
+ const extra = await client.query(
+ "SELECT 1 FROM session_transcript_entries WHERE session_id=$1 AND (seq>$2 OR seq IS NULL OR seq<0) LIMIT 1",
+ [sessionId, tail],
+ );
+ if (extra.rows.length) throw new Error("Canonical history has an extra entry");
+ }
+ if (!source.length) {
+ await client.query("COMMIT");
+ return { busy: false, scanned: 0, changed: 0, afterSeq: options.afterSeq };
+ }
+ const afterSeq = source.at(-1)!.seq;
+ const readCanonical = async () =>
+ (
+ await client.query(
+ "SELECT * FROM session_transcript_entries WHERE session_id=$1 AND seq>$2 AND seq<=$3 ORDER BY seq",
+ [sessionId, options.afterSeq, afterSeq],
+ )
+ ).rows.map(rowToEntry);
+ const before = await readCanonical();
+ const bySeq = new Map(before.map((entry) => [entry.seq, entry]));
+ const sourceSeqs = new Set(source.map((entry) => entry.seq));
+ if (before.some((entry) => !sourceSeqs.has(entry.seq))) throw new Error("Canonical history has an extra entry");
+ const missing = source.filter((entry) => canonicalJson(entry) !== canonicalJson(bySeq.get(entry.seq)));
+ if (options.apply && missing.length) {
+ const next = (
+ await client.query("SELECT COALESCE(MAX(seq),-1)+1 AS seq FROM session_tape WHERE session_id=$1", [sessionId])
+ ).rows[0].seq;
+ const records = missing.map((entry, ordinal) => ({
+ ordinal,
+ payload: jsonbSafeStringify(tapeTranscriptEntryRecord(entry).payload),
+ scope_label: entry.scopeLabel,
+ entry_seq: entry.seq,
+ }));
+ await client.query(
+ `INSERT INTO session_tape(session_id,seq,kind,payload,scope_label,entry_seq,created_at)
+ SELECT $1,$2::int+r.ordinal,'annotation',r.payload,r.scope_label,r.entry_seq,$4
+ FROM jsonb_to_recordset($3::jsonb) AS r(ordinal int,payload text,scope_label text,entry_seq int)`,
+ [sessionId, Number(next), jsonbSafeStringify(records), Date.now()],
+ );
+ if (canonicalJson(await readCanonical()) !== canonicalJson(source))
+ throw new Error("Canonical transcript verification failed");
+ }
+ await client.query("COMMIT");
+ return { busy: false, scanned: source.length, changed: missing.length, afterSeq };
+ } catch (error) {
+ await client.query("ROLLBACK");
+ throw error;
+ }
+}
diff --git a/scripts/migrate-transcript-tape.ts b/scripts/migrate-transcript-tape.ts
new file mode 100644
index 000000000..12d536951
--- /dev/null
+++ b/scripts/migrate-transcript-tape.ts
@@ -0,0 +1,75 @@
+import { parseArgs } from "node:util";
+import pg from "pg";
+import { migrateTranscriptPage } from "./lib/transcript-tape-migration.ts";
+
+const { values } = parseArgs({
+ options: {
+ apply: { type: "boolean", default: false },
+ after: { type: "string", default: "" },
+ "through-created-at": { type: "string", default: String(Date.now()) },
+ "page-size": { type: "string", default: "250" },
+ "max-sessions": { type: "string" },
+ },
+});
+const pageSize = Number(values["page-size"]);
+const through = Number(values["through-created-at"]);
+const maxSessions = values["max-sessions"] === undefined ? Infinity : Number(values["max-sessions"]);
+if (
+ !Number.isInteger(pageSize) ||
+ pageSize < 1 ||
+ pageSize > 1000 ||
+ !Number.isSafeInteger(through) ||
+ through < 0 ||
+ (maxSessions !== Infinity && (!Number.isInteger(maxSessions) || maxSessions < 1))
+)
+ throw new Error("Invalid migration bounds");
+if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL is required");
+const client = new pg.Client({
+ connectionString: process.env.DATABASE_URL,
+ connectionTimeoutMillis: 10_000,
+ application_name: "qm-transcript-tape-migration",
+ options: "-c statement_timeout=30000 -c lock_timeout=1000",
+});
+await client.connect();
+let cursor = values.after!;
+const summary = { sessions: 0, entries: 0, changed: 0, busy: 0 };
+const report = (event: string) =>
+ console.log(JSON.stringify({ event, apply: values.apply, cursor, throughCreatedAt: through, ...summary }));
+report("start");
+try {
+ for (;;) {
+ const batch = (
+ await client.query("SELECT id FROM sessions WHERE id>$1 AND created_at<=$2 ORDER BY id LIMIT $3", [
+ cursor,
+ through,
+ Math.min(100, maxSessions - summary.sessions),
+ ])
+ ).rows;
+ if (!batch.length) break;
+ for (const { id } of batch) {
+ let afterSeq = -1;
+ for (;;) {
+ const page = await migrateTranscriptPage(client, id, { afterSeq, limit: pageSize, apply: values.apply! });
+ if (page.busy) {
+ summary.busy++;
+ console.log(JSON.stringify({ event: "busy", sessionId: id, afterSeq }));
+ break;
+ }
+ summary.entries += page.scanned;
+ summary.changed += page.changed;
+ afterSeq = page.afterSeq;
+ if (page.scanned < pageSize) break;
+ }
+ summary.sessions++;
+ cursor = id;
+ }
+ report("progress");
+ if (summary.sessions >= maxSessions) break;
+ }
+ const partial = summary.sessions >= maxSessions || values.after !== "";
+ report(partial ? "partial" : "complete");
+ process.exitCode = partial ? 2 : 0;
+ if (summary.busy || (!values.apply && summary.changed)) process.exitCode = 1;
+} finally {
+ await client.end();
+}
diff --git a/src/harness/tape-projection.ts b/src/harness/tape-projection.ts
index 17ead3efb..0a76f3b55 100644
--- a/src/harness/tape-projection.ts
+++ b/src/harness/tape-projection.ts
@@ -8,7 +8,6 @@ import { assistantDroppedAtReplay } from "./tape-fold.ts";
import { textFromContent, thinkingBlocksFromContent } from "./pi-harness.ts";
import { swallow } from "../util/errors.ts";
-const UNSERVABLE_MEMO_CAP = 10_000;
const SUFFIX_ROW_CAP_MIN = 100;
const SUFFIX_ROW_CAP_MAX = 5_000;
@@ -202,7 +201,11 @@ export function projectTapeEntries(
tapeRows: readonly TapeRecord[],
opts?: { anchored?: boolean },
): TapeProjection | null {
- const sliced = renderableTapeSlice(tapeRows);
+ const sliced = renderableTapeSlice(
+ tapeRows.filter(
+ (row) => row.kind !== "annotation" || (row.payload as { event?: unknown } | null)?.event !== "transcript_entry",
+ ),
+ );
let rows = sliced;
let base = -1;
if (opts?.anchored) {
@@ -394,7 +397,8 @@ export function projectTapeEntries(
type TranscriptStore = Pick<
SessionStore,
"getEntries" | "visibleEntries" | "getTape" | "latestEntrySeq" | "participantWindowsOf"
->;
+> &
+ Partial<Pick<SessionStore, "getTranscriptEntries">>;
interface TranscriptRead {
entries: SessionEntry[];
@@ -406,19 +410,6 @@ export interface TranscriptSource {
forViewer(sessionId: string, principalId: string, opts?: { limit?: number }): Promise<TranscriptRead>;
}
-function createUnservableMemo() {
- const ids = new Set<string>();
- return {
- has: (sessionId: string): boolean => ids.has(sessionId),
- remember(sessionId: string): void {
- if (ids.size >= UNSERVABLE_MEMO_CAP) ids.clear();
- ids.add(sessionId);
- },
- };
-}
-
-const unservableTapes = createUnservableMemo();
-
interface ProjectedRead {
entries: SessionEntry[];
anchored: boolean;
@@ -427,10 +418,16 @@ interface ProjectedRead {
export function createTranscriptSource(sessions: TranscriptStore): TranscriptSource {
const projected = async (sessionId: string, limit?: number): Promise<ProjectedRead | null> => {
- if (unservableTapes.has(sessionId)) return null;
try {
const latest = await sessions.latestEntrySeq(sessionId);
if (latest < 0) return { entries: [], anchored: false, base: -1 };
+ if (sessions.getTranscriptEntries) {
+ const entries = await sessions.getTranscriptEntries(sessionId, limit === undefined ? undefined : { limit });
+ const first = limit === undefined ? 0 : Math.max(0, latest - limit + 1);
+ if (entries.length === latest - first + 1 && entries.every((entry, i) => entry.seq === first + i)) {
+ return { entries, anchored: first > 0, base: first - 1 };
+ }
+ }
let rows: TapeRecord[];
let anchored = false;
if (limit !== undefined) {
@@ -441,7 +438,6 @@ export function createTranscriptSource(sessions: TranscriptStore): TranscriptSou
rows = await sessions.getTape(sessionId);
}
if (!anchored && tapeHasRenderBlockers(rows)) {
- unservableTapes.remember(sessionId);
return null;
}
const projection = projectTapeEntries(sessionId, rows, { anchored });
diff --git a/src/sessions/memory-session-store.ts b/src/sessions/memory-session-store.ts
index 3966d6303..1ad67d560 100644
--- a/src/sessions/memory-session-store.ts
+++ b/src/sessions/memory-session-store.ts
@@ -42,6 +42,8 @@ import {
sessionCategory,
sessionOrigin,
userMessagePreview,
+ tapeTranscriptEntryRecord,
+ transcriptEntryFromTape,
} from "./session-store.ts";
import { SECURITY_SCREEN_STEP, screenPayloadFromEnvelope } from "../security/security-posture.ts";
@@ -247,7 +249,16 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
scopeLabel: entry.scopeLabel as ScopeId,
createdAt: now(),
};
+ const mirrored = structuredClone(tapeTranscriptEntryRecord(full));
log.push(full);
+ const tapeLog = tape.get(lease.sessionId) ?? [];
+ tapeLog.push({
+ ...mirrored,
+ sessionId: lease.sessionId,
+ seq: tapeLog.length,
+ createdAt: now(),
+ });
+ tape.set(lease.sessionId, tapeLog);
const text = SEARCHABLE_ENTRY_TYPES.has(full.type) ? entrySearchText(full.payload) : null;
if (text?.trim()) {
const index = searchIndex.get(full.sessionId) ?? [];
@@ -265,6 +276,19 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
return opts?.limit !== undefined ? filtered.slice(-opts.limit) : filtered;
},
+ async getTranscriptEntries(sessionId, opts?: GetEntriesOptions) {
+ const projected = new Map<number, SessionEntry>();
+ for (const row of tape.get(sessionId) ?? []) {
+ const entry = transcriptEntryFromTape(row);
+ if (entry) projected.set(entry.seq, entry);
+ }
+ const filtered = [...projected.values()]
+ .filter((entry) => entry.seq >= (opts?.sinceSeq ?? 0))
+ .sort((a, b) => a.seq - b.seq);
+ if (opts?.limit === 0) return [];
+ return opts?.limit === undefined ? filtered : filtered.slice(-opts.limit);
+ },
+
async getContextWindow(sessionId) {
return contextWindowFromEntries(entries.get(sessionId) ?? []);
},
@@ -284,7 +308,16 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore
if (!entry.payload || typeof entry.payload !== "object") continue;
const payload = { ...(entry.payload as Record<string, unknown>) };
delete payload.securityTainted;
+ if (!("securityTainted" in (entry.payload as object))) continue;
entry.payload = payload;
+ const tapeLog = tape.get(sessionId) ?? [];
+ tapeLog.push({
+ ...structuredClone(tapeTranscriptEntryRecord(entry)),
+ sessionId,
+ seq: tapeLog.length,
+ createdAt: now(),
+ });
+ tape.set(sessionId, tapeLog);
}
return true;
},
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts
index f5149a763..cbf6b571a 100644
--- a/src/sessions/postgres-session-store.ts
+++ b/src/sessions/postgres-session-store.ts
@@ -42,6 +42,7 @@ import {
stableOriginPattern,
threadRefCronIdExpr,
userMessagePreview,
+ tapeTranscriptEntryRecord,
} from "./session-store.ts";
import { SECURITY_SCREEN_STEP, screenPayloadFromEnvelope } from "../security/security-posture.ts";
@@ -156,7 +157,7 @@ function rowToParticipantWindow(r: Record<string, unknown>): ParticipantWindow {
};
}
-function rowToEntry(r: Record<string, unknown>): SessionEntry {
+export function rowToEntry(r: Record<string, unknown>): SessionEntry {
return {
sessionId: r.session_id as string,
seq: Number(r.seq),
@@ -534,6 +535,25 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
`ALTER TABLE session_tape ADD COLUMN IF NOT EXISTS source_role TEXT`,
],
},
+ {
+ id: "sessions/store/0017-transcript-entries",
+ statements: [
+ `CREATE INDEX IF NOT EXISTS session_tape_transcript_entries
+ ON session_tape(session_id, entry_seq DESC, seq DESC)
+ WHERE kind = 'annotation' AND safe_json(payload)->>'event' = 'transcript_entry'`,
+ `CREATE OR REPLACE VIEW session_transcript_entries AS
+ SELECT DISTINCT ON (session_id, entry_seq)
+ session_id, entry_seq AS seq,
+ (safe_json(payload)->'entry'->>'parentSeq')::int AS parent_seq,
+ safe_json(payload)->'entry'->>'type' AS type,
+ (safe_json(payload)->'entry'->'payload')::text AS payload,
+ scope_label,
+ (safe_json(payload)->'entry'->>'at')::bigint AS created_at
+ FROM session_tape t
+ WHERE kind = 'annotation' AND safe_json(payload)->>'event' = 'transcript_entry'
+ ORDER BY t.session_id, t.entry_seq DESC, t.seq DESC`,
+ ],
+ },
],
[
{
@@ -763,6 +783,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
"INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7)",
[full.sessionId, full.seq, full.parentSeq, full.type, stored, full.scopeLabel, full.createdAt],
);
+ await insertTapeRow(client, full.sessionId, tapeTranscriptEntryRecord(full));
await client.query(
`UPDATE sessions
SET last_activity = CASE WHEN last_activity >= $2::bigint - ${LAST_ACTIVITY_DEBOUNCE_MS}
@@ -786,13 +807,28 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
},
async clearSecurityTaint(sessionId) {
- const updated = await q(
- "UPDATE session_entries SET payload = (payload::jsonb - 'securityTainted')::text " +
- "WHERE session_id = $1 AND payload LIKE '%\"securityTainted\"%' RETURNING 1",
- [sessionId],
+ return withPgTransaction(await pool(), async (client) => {
+ await lockSession(client, sessionId);
+ const updated = await client.query(
+ "UPDATE session_entries SET payload = (payload::jsonb - 'securityTainted')::text " +
+ "WHERE session_id = $1 AND jsonb_typeof(payload::jsonb) = 'object' AND payload::jsonb ? 'securityTainted' RETURNING *",
+ [sessionId],
+ );
+ for (const row of updated.rows) {
+ await insertTapeRow(client, sessionId, tapeTranscriptEntryRecord(rowToEntry(row)));
+ }
+ if (updated.rows.length > 0) return true;
+ return (await client.query("SELECT 1 FROM sessions WHERE id = $1", [sessionId])).rows.length === 1;
+ });
+ },
+
+ async getTranscriptEntries(sessionId, opts?: GetEntriesOptions) {
+ const rows = await q(
+ "SELECT * FROM session_transcript_entries WHERE session_id = $1 AND seq >= $2 ORDER BY seq DESC" +
+ (opts?.limit === undefined ? "" : " LIMIT $3"),
+ [sessionId, opts?.sinceSeq ?? 0, ...(opts?.limit === undefined ? [] : [opts.limit])],
);
- if (updated.length > 0) return true;
- return (await q("SELECT 1 FROM sessions WHERE id = $1", [sessionId])).length === 1;
+ return rows.map(rowToEntry).reverse();
},
async appendTape(lease, rec: NewTapeRecord): Promise<TapeRecord> {
diff --git a/src/sessions/session-store.ts b/src/sessions/session-store.ts
index 8109ed653..2e4c3d03d 100644
--- a/src/sessions/session-store.ts
+++ b/src/sessions/session-store.ts
@@ -180,6 +180,44 @@ export function tapeEntryMirrorRecord(entry: {
};
}
+export function tapeTranscriptEntryRecord(entry: SessionEntry): NewTapeRecord {
+ return {
+ kind: "annotation",
+ payload: {
+ event: "transcript_entry",
+ entry: { type: entry.type, payload: entry.payload, at: entry.createdAt, parentSeq: entry.parentSeq },
+ },
+ scopeLabel: entry.scopeLabel,
+ entrySeq: entry.seq,
+ };
+}
+
+export function transcriptEntryFromTape(row: TapeRecord): SessionEntry | null {
+ if (row.kind !== "annotation" || row.entrySeq === undefined) return null;
+ const payload = row.payload as {
+ event?: unknown;
+ entry?: { type?: unknown; payload?: unknown; at?: unknown; parentSeq?: unknown };
+ } | null;
+ const entry = payload?.entry;
+ if (
+ payload?.event !== "transcript_entry" ||
+ !entry ||
+ typeof entry.type !== "string" ||
+ typeof entry.at !== "number" ||
+ (entry.parentSeq !== null && typeof entry.parentSeq !== "number")
+ )
+ return null;
+ return {
+ sessionId: row.sessionId,
+ seq: row.entrySeq,
+ parentSeq: entry.parentSeq,
+ type: entry.type as EntryType,
+ payload: entry.payload ?? null,
+ scopeLabel: row.scopeLabel,
+ createdAt: entry.at,
+ };
+}
+
export type TranscriptAppendSessions = Pick<SessionStore, "append" | "appendTape" | "latestEntrySeq" | "tapeCoverage">;
export async function appendEntryOutsideTurn(
@@ -671,6 +709,7 @@ export interface SessionStore {
append(lease: Lease, entry: NewEntry): Promise<SessionEntry>;
getEntries(sessionId: string, opts?: GetEntriesOptions): Promise<SessionEntry[]>;
+ getTranscriptEntries(sessionId: string, opts?: GetEntriesOptions): Promise<SessionEntry[]>;
getContextWindow(sessionId: string): Promise<ContextWindow>;
getEntry(sessionId: string, seq: number): Promise<SessionEntry | undefined>;
latestEntrySeq(sessionId: string): Promise<number>;
diff --git a/test/message-revisions.test.ts b/test/message-revisions.test.ts
index 2760b2e08..ead3a465f 100644
--- a/test/message-revisions.test.ts
+++ b/test/message-revisions.test.ts
@@ -201,7 +201,7 @@ test("a channel thread reply's edit reaches the thread session via the sub root"
assert.equal(marks[0]!.name, "Josh");
});
-test("when the tape is not contiguous the marker entry still lands but the tape stays untouched", async () => {
+test("when model tape is not contiguous the revision is mirrored without advancing model coverage", async () => {
const sessions = createMemorySessionStore();
const session = await sessions.getOrCreateByThread(`dm:${DM}`, "dm", SCOPE, undefined, "slack");
const { lease } = await sessions.acquireLease(session.id, "turn");
@@ -211,7 +211,13 @@ test("when the tape is not contiguous the marker entry still lands but the tape
await recordMessageRevisions(sessions, [edit()]);
assert.equal(revisions(await sessions.getEntries(session.id)).length, 1);
- assert.equal((await sessions.getTape(session.id)).length, 0);
+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), await sessions.getEntries(session.id));
+ assert.equal(await sessions.tapeCoverage(session.id), -1);
+ assert.ok(
+ (await sessions.getTape(session.id)).every(
+ (row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry",
+ ),
+ );
});
test("the marker renders into model context on the entries-replay path", () => {
diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts
index dfcc40392..059173c94 100644
--- a/test/postgres-store.test.ts
+++ b/test/postgres-store.test.ts
@@ -1,4 +1,5 @@
import "./run-availability.ts";
+import { migrateTranscriptPage } from "../scripts/lib/transcript-tape-migration.ts";
import { test, before } from "node:test";
import { randomUUID } from "node:crypto";
import assert from "node:assert/strict";
@@ -380,8 +381,9 @@ test("pg latestEntrySeq, participant windows, and tape meta attachments round-tr
meta: { bareText: "one", attachments, sourceRole: "agent" },
});
const rows = await s.getTape(session.id);
- assert.deepEqual(rows[0]!.meta?.attachments, attachments);
- assert.equal(rows[0]!.meta?.sourceRole, "agent");
+ const message = rows.find((row) => row.kind === "message")!;
+ assert.deepEqual(message.meta?.attachments, attachments);
+ assert.equal(message.meta?.sourceRole, "agent");
});
test("pg participant tenure remains exact when every event shares a timestamp", { skip }, async () => {
@@ -2234,3 +2236,197 @@ test("pg session store: bounded participant listing is recent and optional", { s
);
assert.deepEqual(await store.listByParticipant("bounded-user", { limit: 0 }), []);
});
+
+test(
+ "pg canonical transcript annotations are exact and taint release preserves original identity",
+ { skip },
+ async () => {
+ const s = createPostgresSessionStore(URL!);
+ const scope = scopeId("personal", "canonical-tape");
+ const session = await s.getOrCreateByThread("pg-canonical-tape", "dm", scope);
+ const { lease } = await s.acquireLease(session.id);
+ assert.ok(lease);
+ for (let i = 0; i < 4; i++)
+ await s.append(lease, {
+ type: "tool_result",
+ scopeLabel: scope,
+ payload: { tool: "execute", callId: `c${i}`, result: "denied", isError: true, code: 1, securityTainted: true },
+ });
+ const before = await s.getEntries(session.id);
+ assert.deepEqual(await s.getTranscriptEntries(session.id), before);
+ assert.deepEqual(await s.getTranscriptEntries(session.id, { limit: 2 }), before.slice(-2));
+ assert.deepEqual(await s.getTranscriptEntries(session.id, { sinceSeq: 1, limit: 2 }), before.slice(-2));
+ assert.equal(await s.tapeCoverage(session.id), -1);
+ assert.equal(await s.clearSecurityTaint(session.id), true);
+ const after = await s.getEntries(session.id);
+ assert.deepEqual(await s.getTranscriptEntries(session.id), after);
+ assert.equal(after[0]!.createdAt, before[0]!.createdAt);
+ assert.equal(after[0]!.parentSeq, before[0]!.parentSeq);
+ assert.equal((after[0]!.payload as { securityTainted?: boolean }).securityTainted, undefined);
+ assert.equal((await s.getTape(session.id)).length, 8);
+ assert.equal(await s.clearSecurityTaint(session.id), true);
+ assert.equal((await s.getTape(session.id)).length, 8);
+ assert.equal(await s.tapeCoverage(session.id), -1);
+ await s.releaseLease(lease);
+ },
+);
+
+test("pg transcript write failure rolls back the entry and its counters", { skip }, async () => {
+ const s = createPostgresSessionStore(URL!);
+ const scope = scopeId("personal", "canonical-failure");
+ const session = await s.getOrCreateByThread("pg-canonical-failure", "dm", scope);
+ const { lease } = await s.acquireLease(session.id);
+ assert.ok(lease);
+ const pg = (await import("pg")).default;
+ const client = new pg.Client({ connectionString: URL! });
+ await client.connect();
+ await client.query(
+ `CREATE FUNCTION reject_transcript_test() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'transcript unavailable'; END $$`,
+ );
+ await client.query(
+ `CREATE TRIGGER reject_transcript_test BEFORE INSERT ON session_tape FOR EACH ROW WHEN (NEW.session_id = '${session.id}') EXECUTE FUNCTION reject_transcript_test()`,
+ );
+ try {
+ await assert.rejects(
+ s.append(lease, { type: "user", payload: { text: "must rollback" }, scopeLabel: scope }),
+ /transcript unavailable/,
+ );
+ assert.deepEqual(await s.getEntries(session.id), []);
+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);
+ assert.equal(await s.latestEntrySeq(session.id), -1);
+ const counters = (await client.query("SELECT messages FROM sessions WHERE id=$1", [session.id])).rows[0];
+ assert.equal(counters.messages, 0);
+ } finally {
+ await client.query("DROP TRIGGER reject_transcript_test ON session_tape");
+ await client.query("DROP FUNCTION reject_transcript_test()");
+ await client.end();
+ await s.releaseLease(lease);
+ }
+});
+
+test("pg transcript backfill is bounded, idempotent, and independent of model coverage", { skip }, async () => {
+ const s = createPostgresSessionStore(URL!);
+ const scope = scopeId("personal", "canonical-backfill");
+ const session = await s.getOrCreateByThread("pg-canonical-backfill", "dm", scope);
+ const pg = (await import("pg")).default;
+ const client = new pg.Client({ connectionString: URL! });
+ await client.connect();
+ try {
+ await client.query(
+ `INSERT INTO session_entries(session_id,seq,parent_seq,type,payload,scope_label,created_at)
+ SELECT $1,i,NULLIF(i-1,-1),'user',json_build_object('text','history '||i,'securityTainted',true)::text,$2,123000+i
+ FROM generate_series(0,619) i`,
+ [session.id, scope],
+ );
+ const dry = await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 200, apply: false });
+ assert.deepEqual(dry, { busy: false, scanned: 200, changed: 200, afterSeq: 199 });
+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);
+ for (let afterSeq = -1; afterSeq < 619;) {
+ const page = await migrateTranscriptPage(client, session.id, { afterSeq, limit: 200, apply: true });
+ assert.ok(!page.busy);
+ assert.ok(page.scanned <= 200);
+ afterSeq = page.afterSeq;
+ }
+ assert.deepEqual(await s.getTranscriptEntries(session.id), await s.getEntries(session.id));
+ assert.equal(await s.tapeCoverage(session.id), -1);
+ const repeated = await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 200, apply: true });
+ assert.deepEqual(repeated, { busy: false, scanned: 200, changed: 0, afterSeq: 199 });
+ assert.equal((await s.getTape(session.id)).length, 620);
+ await s.clearSecurityTaint(session.id);
+ assert.deepEqual(await s.getTranscriptEntries(session.id), await s.getEntries(session.id));
+ } finally {
+ await client.end();
+ }
+});
+
+test("pg transcript backfill refuses an active lease or writer transaction", { skip }, async () => {
+ const s = createPostgresSessionStore(URL!);
+ const scope = scopeId("personal", "canonical-busy");
+ const session = await s.getOrCreateByThread("pg-canonical-busy", "dm", scope);
+ const pg = (await import("pg")).default;
+ const client = new pg.Client({ connectionString: URL! });
+ const writer = new pg.Client({ connectionString: URL! });
+ await client.connect();
+ await writer.connect();
+ try {
+ const { lease } = await s.acquireLease(session.id);
+ assert.ok(lease);
+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {
+ busy: true,
+ });
+ await s.releaseLease(lease);
+ await writer.query("BEGIN");
+ await writer.query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))", [session.id]);
+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {
+ busy: true,
+ });
+ await writer.query("ROLLBACK");
+ assert.deepEqual(await migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply: true }), {
+ busy: false,
+ scanned: 0,
+ changed: 0,
+ afterSeq: -1,
+ });
+ } finally {
+ await writer.end();
+ await client.end();
+ }
+});
+
+test(
+ "pg transcript migration rejects extra canonical tails and entries in an empty legacy session",
+ { skip },
+ async () => {
+ const s = createPostgresSessionStore(URL!);
+ const pg = (await import("pg")).default;
+ const client = new pg.Client({ connectionString: URL! });
+ await client.connect();
+ try {
+ for (const originalCount of [0, 3]) {
+ const scope = scopeId("personal", `extra-canonical-${originalCount}`);
+ const session = await s.getOrCreateByThread(`pg-extra-canonical-${originalCount}`, "dm", scope);
+ const { lease } = await s.acquireLease(session.id);
+ assert.ok(lease);
+ for (let i = 0; i <= originalCount; i++)
+ await s.append(lease, { type: "user", payload: { text: `row ${i}` }, scopeLabel: scope });
+ await s.releaseLease(lease);
+ await client.query("DELETE FROM session_entries WHERE session_id=$1 AND seq=$2", [session.id, originalCount]);
+ for (const apply of [false, true]) {
+ await assert.rejects(
+ migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply }),
+ /extra entry/,
+ );
+ }
+ assert.equal((await s.getTranscriptEntries(session.id)).length, originalCount + 1);
+ }
+ } finally {
+ await client.end();
+ }
+ },
+);
+
+test("pg transcript migration refuses gapped legacy history", { skip }, async () => {
+ const s = createPostgresSessionStore(URL!);
+ const pg = (await import("pg")).default;
+ const client = new pg.Client({ connectionString: URL! });
+ await client.connect();
+ try {
+ const scope = scopeId("personal", "gapped-transcript");
+ const session = await s.getOrCreateByThread("pg-gapped-transcript", "dm", scope);
+ const { lease } = await s.acquireLease(session.id);
+ assert.ok(lease);
+ for (let i = 0; i < 3; i++)
+ await s.append(lease, { type: "user", payload: { text: `row ${i}` }, scopeLabel: scope });
+ await s.releaseLease(lease);
+ await client.query("DELETE FROM session_entries WHERE session_id=$1 AND seq=1", [session.id]);
+ await client.query("DELETE FROM session_tape WHERE session_id=$1", [session.id]);
+ for (const apply of [false, true])
+ await assert.rejects(
+ migrateTranscriptPage(client, session.id, { afterSeq: -1, limit: 10, apply }),
+ /sequence gap/,
+ );
+ assert.deepEqual(await s.getTranscriptEntries(session.id), []);
+ } finally {
+ await client.end();
+ }
+});
diff --git a/test/run-result-delivery.test.ts b/test/run-result-delivery.test.ts
index 2720c24d5..c8252467a 100644
--- a/test/run-result-delivery.test.ts
+++ b/test/run-result-delivery.test.ts
@@ -281,8 +281,14 @@ test("a parked run's failure lands as a turn_failure entry in the run's own sess
runId: "r-1",
});
const tape = await sessions.getTape(session.id);
- assert.equal(tape.length, 1, "a turnEnd checkpoint keeps the tape projection servable");
- assert.equal(tape[0]!.kind, "annotation");
+ assert.equal(tape.length, 2);
+ assert.equal(
+ tape.filter((row) => (row.payload as { turnEnd?: boolean }).turnEnd === true).length,
+ 1,
+ "exactly one turnEnd checkpoint keeps model coverage intact",
+ );
+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), entries);
+ assert.equal(await sessions.tapeCoverage(session.id), entries.at(-1)!.seq);
assert.equal(await recordRunFailureEntry(sessions, failedRun()), false, "recording is idempotent");
assert.equal((await sessions.getEntries(session.id)).length, 1);
diff --git a/test/session-view-bounds.test.ts b/test/session-view-bounds.test.ts
index 517ea137c..0347af096 100644
--- a/test/session-view-bounds.test.ts
+++ b/test/session-view-bounds.test.ts
@@ -52,6 +52,17 @@ test("a tailTurns view is bounded yet reports the same earlierEntries as the ful
}
});
+function withoutCanonicalTranscript(sessions: SessionStore) {
+ const getTape = sessions.getTape.bind(sessions);
+ sessions.getTranscriptEntries = async () => [];
+ sessions.getTape = async (sessionId, opts) => {
+ const rows = (await getTape(sessionId)).filter(
+ (row) => (row.payload as { event?: string }).event !== "transcript_entry" && row.seq > (opts?.sinceSeq ?? -1),
+ );
+ return opts?.limit === undefined ? rows : rows.slice(-opts.limit);
+ };
+}
+
async function coarseForeignSession(sessions: SessionStore, turns = 1) {
const scope = scopeId("personal", "U1");
const session = await sessions.getOrCreateByThread("web:U1:coarse-pins", "dm", scope, undefined, "web");
@@ -103,6 +114,7 @@ test("bounded earlierEntries on a coarse foreign session counts renderable entri
const built = freshApp();
built.runtime.start();
try {
+ withoutCanonicalTranscript(built.sessions);
const { session } = await coarseForeignSession(built.sessions, 30);
const full = (await built.app.getSession(session.id))!;
assert.equal(full.earlierEntries ?? 0, 0, "the unbounded read reports nothing earlier");
@@ -133,6 +145,7 @@ test("pins on entries a coarse projection drops still resolve through the target
const built = freshApp();
built.runtime.start();
try {
+ withoutCanonicalTranscript(built.sessions);
const { session, narration } = await coarseForeignSession(built.sessions);
const view = (await built.app.getSessionForViewer(session.id, "U1"))!;
assert.deepEqual(
@@ -153,3 +166,22 @@ test("pins on entries a coarse projection drops still resolve through the target
await built.runtime.stop();
}
});
+
+test("canonical transcripts retain foreign harness tool detail and exact bounded counts", async () => {
+ const built = freshApp();
+ built.runtime.start();
+ try {
+ const { session, narration } = await coarseForeignSession(built.sessions, 30);
+ const full = (await built.app.getSessionForViewer(session.id, "U1", { tailTurns: 9999 }))!;
+ assert.deepEqual(full.entries, await built.sessions.getEntries(session.id));
+ assert.equal(full.entries.length, 150);
+ const bounded = (await built.app.getSessionForViewer(session.id, "U1", { tailTurns: 1 }))!;
+ assert.deepEqual(bounded.entries, full.entries.slice(-5));
+ assert.equal(bounded.earlierEntries, 145);
+ const pinned = await built.app.pinConversationItem("web:U1:coarse-pins", "U1", { entrySeq: narration.seq });
+ assert.ok("pin" in pinned && pinned.pin);
+ assert.equal(pinned.pin.preview, "queue narration detail");
+ } finally {
+ await built.runtime.stop();
+ }
+});
diff --git a/test/tape-canonical-transcript.test.ts b/test/tape-canonical-transcript.test.ts
new file mode 100644
index 000000000..456f6ba05
--- /dev/null
+++ b/test/tape-canonical-transcript.test.ts
@@ -0,0 +1,119 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
+import { createTranscriptSource } from "../src/harness/tape-projection.ts";
+import { foldTape } from "../src/harness/tape-fold.ts";
+import { tapeTranscriptEntryRecord } from "../src/sessions/session-store.ts";
+import type { ScopeId } from "../src/types.ts";
+
+const scope = "personal:[email protected]" as ScopeId;
+
+async function scenario() {
+ const store = createMemorySessionStore();
+ const session = await store.getOrCreateByThread("canonical-transcript", "dm", scope);
+ const { lease } = await store.acquireLease(session.id);
+ assert.ok(lease);
+ return { store, session, lease };
+}
+
+test("exact tape annotations retain tool fields without advancing model coverage", async () => {
+ const { store, session, lease } = await scenario();
+ const entry = await store.append(lease, {
+ type: "tool_result",
+ scopeLabel: scope,
+ payload: {
+ tool: "execute",
+ callId: "failed",
+ stdout: "",
+ stderr: "denied",
+ code: 1,
+ isError: true,
+ result: "denied",
+ },
+ });
+ assert.deepEqual(await store.getTranscriptEntries(session.id), [entry]);
+ assert.equal(await store.tapeCoverage(session.id), -1);
+ assert.deepEqual(foldTape(await store.getTape(session.id)), []);
+ const source = createTranscriptSource({
+ ...store,
+ getEntries: async () => {
+ throw new Error("legacy read");
+ },
+ });
+ assert.deepEqual((await source.forRender(session.id)).entries, [entry]);
+});
+
+test("taint release supersedes the exact annotation without changing identity or native model history", async () => {
+ const { store, session, lease } = await scenario();
+ await store.append(lease, { type: "user", payload: { text: "reviewed", securityTainted: true }, scopeLabel: scope });
+ const before = await store.getTranscriptEntries(session.id);
+ const beforeRows = await store.getTape(session.id);
+ assert.equal(await store.clearSecurityTaint(session.id), true);
+ assert.deepEqual(await store.getTranscriptEntries(session.id), await store.getEntries(session.id));
+ assert.deepEqual(await store.getTranscriptEntries(session.id), [{ ...before[0]!, payload: { text: "reviewed" } }]);
+ assert.equal((await store.getTape(session.id)).length, beforeRows.length + 1);
+ assert.equal(await store.tapeCoverage(session.id), -1);
+ assert.deepEqual(foldTape(await store.getTape(session.id)), []);
+ assert.equal(await store.clearSecurityTaint(session.id), true);
+ assert.equal((await store.getTape(session.id)).length, beforeRows.length + 1);
+});
+
+test("canonical transcript reads preserve full histories, bounded tails and participant windows", async () => {
+ const { store, session, lease } = await scenario();
+ for (let i = 0; i < 620; i++) {
+ if (i === 300) await store.addParticipant(session.id, "viewer");
+ if (i === 550) await store.removeParticipant(session.id, "viewer");
+ await store.append(lease, { type: "user", payload: { text: `message ${i}` }, scopeLabel: scope });
+ }
+ const entries = await store.getEntries(session.id);
+ const visible = await store.visibleEntries(session.id, "viewer");
+ const calls: Array<number | undefined> = [];
+ const source = createTranscriptSource({
+ ...store,
+ getEntries: async () => {
+ throw new Error("legacy read");
+ },
+ visibleEntries: async () => {
+ throw new Error("legacy viewer read");
+ },
+ getTape: async () => {
+ throw new Error("native tape read");
+ },
+ getTranscriptEntries: (id, opts) => {
+ calls.push(opts?.limit);
+ return store.getTranscriptEntries(id, opts);
+ },
+ });
+ assert.deepEqual((await source.forRender(session.id)).entries, entries);
+ assert.deepEqual(await source.forRender(session.id, { limit: 6 }), { entries: entries.slice(-6), earlier: 614 });
+ assert.equal(calls.at(-1), 6);
+ assert.deepEqual((await source.forRender(session.id, { sinceSeq: 615 })).entries, entries.slice(615));
+ assert.deepEqual((await source.forViewer(session.id, "viewer")).entries, visible);
+ assert.deepEqual((await source.forViewer(session.id, "stranger")).entries, []);
+});
+
+test("partial canonical coverage uses legacy history until every sequence is present", async () => {
+ const { store, session, lease } = await scenario();
+ for (let i = 0; i < 3; i++) await store.append(lease, { type: "user", payload: { text: `${i}` }, scopeLabel: scope });
+ const entries = await store.getEntries(session.id);
+ let reads = 0;
+ const source = createTranscriptSource({
+ ...store,
+ getTranscriptEntries: async () => entries.slice(1),
+ getTape: async () => [],
+ getEntries: async () => {
+ reads++;
+ return entries;
+ },
+ });
+ assert.deepEqual((await source.forRender(session.id)).entries, entries);
+ assert.equal(reads, 1);
+});
+
+test("historical canonical annotations preserve original parent, scope and timestamp", async () => {
+ const { store, session, lease } = await scenario();
+ const entry = await store.append(lease, { type: "user", payload: { text: "original" }, scopeLabel: scope });
+ const historical = { ...entry, parentSeq: null, createdAt: 123, scopeLabel: "org:historic" as ScopeId };
+ await store.appendTape(lease, tapeTranscriptEntryRecord(historical));
+ assert.deepEqual(await store.getTranscriptEntries(session.id), [historical]);
+});
diff --git a/test/tape-projection.test.ts b/test/tape-projection.test.ts
index 93a1a990c..62f92f68e 100644
--- a/test/tape-projection.test.ts
+++ b/test/tape-projection.test.ts
@@ -22,7 +22,7 @@ interface Sim {
}
async function simSession(threadRef = "dm:projection-test"): Promise<Sim> {
- const store = createMemorySessionStore({ now: () => CLOCK });
+ const store = { ...createMemorySessionStore({ now: () => CLOCK }), getTranscriptEntries: async () => [] };
const session = await store.getOrCreateByThread(threadRef, "dm", scope);
const { lease } = await store.acquireLease(session.id);
assert.ok(lease);
diff --git a/test/tape-retirement.test.ts b/test/tape-retirement.test.ts
index e83f611a9..fa3c67199 100644
--- a/test/tape-retirement.test.ts
+++ b/test/tape-retirement.test.ts
@@ -395,7 +395,11 @@ test("a tainted uncovered session is refused without a coverage claim", async ()
const entries = await sim.store.getEntries(sim.session.id);
const outcome = await appendRenderImport(sim.store, sim.lease, entries, scope, true);
assert.equal(outcome, "unservable-fold");
- assert.equal((await sim.store.getTape(sim.session.id)).length, 0);
+ assert.ok(
+ (await sim.store.getTape(sim.session.id)).every(
+ (row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry",
+ ),
+ );
assert.equal(await sim.store.tapeCoverage(sim.session.id), -1);
});
diff --git a/test/web-transcript-delivery.test.ts b/test/web-transcript-delivery.test.ts
index 454cd755c..6db325d17 100644
--- a/test/web-transcript-delivery.test.ts
+++ b/test/web-transcript-delivery.test.ts
@@ -116,7 +116,11 @@ test("a session whose tape is already behind gets the entry but no orphan tape r
const entries = await sessions.getEntries(session.id);
assert.equal(entries.length, 2, "the transcript entry still lands");
const tape = await sessions.getTape(session.id);
- assert.equal(tape.length, 0, "no tape rows without their covering annotation (heal owns this session)");
+ assert.ok(
+ tape.every((row) => row.kind === "annotation" && (row.payload as { event?: string }).event === "transcript_entry"),
+ );
+ assert.equal(await sessions.tapeCoverage(session.id), -1, "transcript annotations do not advance model coverage");
+ assert.deepEqual(await sessions.getTranscriptEntries(session.id), entries);
});
test("repeated drains and a restarted decorator never duplicate the transcript entry", async () => {