Multiplayer agent harness for work.
Drop the legacy session history tables (#2486)
Regan Belld77a8b598d74
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex 729202689..f0abd7845 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -894,6 +894,22 @@ export function createPostgresSessionStore(connectionString: string, opts: Storestatements: [],},...ITEMS_CUTOVER_MIGRATIONS,+ {+ id: "sessions/store/0037-drop-legacy-history",+ statements: [+ `SET LOCAL lock_timeout = '60s'`,+ `DROP TRIGGER IF EXISTS session_entry_search ON items`,+ `DROP FUNCTION IF EXISTS sync_item_search(), sync_session_entry_search()`,+ `DROP VIEW IF EXISTS session_transcript_entries`,+ `DROP TABLE IF EXISTS session_entry_search, legacy_session_entries, session_entries, session_tape,+ legacy_tape_steps, legacy_turn_runs, legacy_tape_steps_progress, legacy_tape_skipped,+ legacy_tape_writes, legacy_prep_runs`,+ `DROP PROCEDURE IF EXISTS prepare_legacy_tape_steps(INT)`,+ `DROP FUNCTION IF EXISTS legacy_tape_written(), legacy_tape_steps_batch(TEXT[], BIGINT),+ entry_search_text(text), items_copy_payload(text), items_copy_payload(jsonb)`,+ `DROP SCHEMA IF EXISTS items_copy_ops CASCADE`,+ ],+ },],[{@@ -907,17 +923,6 @@ export function createPostgresSessionStore(connectionString: string, opts: StoreEND $safe_jsonb_parallel$`,],},- {- id: "sessions/maintenance/entry-search-text-parallel",- statements: [- `DO $entry_search_text_parallel$- BEGIN- IF to_regprocedure('entry_search_text(text)') IS NOT NULL THEN- ALTER FUNCTION entry_search_text(text) PARALLEL UNSAFE;- END IF;- END $entry_search_text_parallel$`,- ],- },],);diff --git a/test/postgres-items-migration.test.ts b/test/postgres-items-migration.test.tsindex 54b547f0d..6c59a3d87 100644--- a/test/postgres-items-migration.test.ts+++ b/test/postgres-items-migration.test.ts@@ -8,129 +8,146 @@ import { scopeId, type EntryType, type SessionEntry } from "../src/types.ts";const URL = process.env.DATABASE_URL;const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the items upgrade test";-test("pg upgrade from a pre-items release: startup migrations copy history into items", { skip }, async () => {- const pg = (await import("pg")).default;- const raw = new pg.Pool({ connectionString: URL });- try {- createPostgresSessionStore(URL!);- const migrations = registeredPgMigrations(URL!);- await applyPgMigrations(- raw,- migrations.filter((migration) => migration.id < "sessions/store/0035-items"),- );- const sessionId = randomUUID();- const scope = scopeId("personal", "alice");- const runId = randomUUID();- const legacy: [EntryType, Record<string, unknown>][] = [- ["user", { text: "hello", name: "Alice", principalId: "alice", runId }],- ["thinking", { text: "hmm", thinkingSignature: "sig" }],- ["text", { text: "narrating" }],- ["tool_call", { tool: "execute", callId: "c1", input: { command: "ls" } }],- ["tool_result", { tool: "execute", callId: "c1", result: "out\n", stdout: "out\n" }],- ["assistant", { text: "final answer" }],- ["system", { kind: "goal", goal: { objective: "x", status: "active" } }],- ["assistant", { text: "cron says hi", deliveryKey: "cron:1", via: "cron" }],- ["delivery", { text: "a.txt (text/plain, 3 bytes)", files: ["a.txt"] }],- ["user", { text: "steer", steered: true }],- ["system", { kind: "turn_failure", message: "boom", runId }],- ["system", { kind: "wake", cause: "cron", text: "cron fired" }],- ["soul" as EntryType, { text: "you are" }],- ["text_start" as EntryType, { phase: "final_answer" }],- ["system", { kind: "goal_cancelled", objective: "Example goal", cancelledBy: "alice" }],- ];- await raw.query(- "INSERT INTO sessions(id, type, scope_id, thread_ref, created_at, messages) VALUES ($1, 'dm', $2, 'web:cutover', 1, $3)",- [sessionId, scope, legacy.length],- );- await raw.query(- "INSERT INTO participants(session_id, principal_id, valid_from, valid_from_seq) VALUES ($1, 'alice', 0, 0)",- [sessionId],- );- const legacyEntries: SessionEntry[] = [];- for (const [seq, [type, payload]] of legacy.entries()) {+test(+ "pg upgrade from a pre-items release: startup migrations copy history into items, then drop the legacy tables",+ { skip },+ async () => {+ const pg = (await import("pg")).default;+ const raw = new pg.Pool({ connectionString: URL });+ try {+ createPostgresSessionStore(URL!);+ const migrations = registeredPgMigrations(URL!);+ await applyPgMigrations(+ raw,+ migrations.filter((migration) => migration.id < "sessions/store/0035-items"),+ );+ const sessionId = randomUUID();+ const scope = scopeId("personal", "alice");+ const runId = randomUUID();+ const legacy: [EntryType, Record<string, unknown>][] = [+ ["user", { text: "hello", name: "Alice", principalId: "alice", runId }],+ ["thinking", { text: "hmm", thinkingSignature: "sig" }],+ ["text", { text: "narrating" }],+ ["tool_call", { tool: "execute", callId: "c1", input: { command: "ls" } }],+ ["tool_result", { tool: "execute", callId: "c1", result: "out\n", stdout: "out\n" }],+ ["assistant", { text: "final answer" }],+ ["system", { kind: "goal", goal: { objective: "x", status: "active" } }],+ ["assistant", { text: "cron says hi", deliveryKey: "cron:1", via: "cron" }],+ ["delivery", { text: "a.txt (text/plain, 3 bytes)", files: ["a.txt"] }],+ ["user", { text: "steer", steered: true }],+ ["system", { kind: "turn_failure", message: "boom", runId }],+ ["system", { kind: "wake", cause: "cron", text: "cron fired" }],+ ["soul" as EntryType, { text: "you are" }],+ ["text_start" as EntryType, { phase: "final_answer" }],+ ["system", { kind: "goal_cancelled", objective: "Example goal", cancelledBy: "alice" }],+ ];await raw.query(- `INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at)- VALUES ($1, $2, $3, $4, $5, $6, $7)`,- [sessionId, seq, seq ? seq - 1 : null, type, JSON.stringify(payload), scope, 1_000 + seq],+ "INSERT INTO sessions(id, type, scope_id, thread_ref, created_at, messages) VALUES ($1, 'dm', $2, 'web:cutover', 1, $3)",+ [sessionId, scope, legacy.length],);- legacyEntries.push({- sessionId,- seq,- parentSeq: seq ? seq - 1 : null,- type,- payload,- scopeLabel: scope,- createdAt: 1_000 + seq,- });- }+ await raw.query(+ "INSERT INTO participants(session_id, principal_id, valid_from, valid_from_seq) VALUES ($1, 'alice', 0, 0)",+ [sessionId],+ );+ const legacyEntries: SessionEntry[] = [];+ for (const [seq, [type, payload]] of legacy.entries()) {+ await raw.query(+ `INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at)+ VALUES ($1, $2, $3, $4, $5, $6, $7)`,+ [sessionId, seq, seq ? seq - 1 : null, type, JSON.stringify(payload), scope, 1_000 + seq],+ );+ legacyEntries.push({+ sessionId,+ seq,+ parentSeq: seq ? seq - 1 : null,+ type,+ payload,+ scopeLabel: scope,+ createdAt: 1_000 + seq,+ });+ }- await applyPgMigrations(raw, migrations);- await applyPgMigrations(raw, migrations);+ await applyPgMigrations(raw, migrations);+ await applyPgMigrations(raw, migrations);- const store = createPostgresSessionStore(URL!);- const copied = await store.getEntries(sessionId);- assert.deepEqual(copied.at(-1)?.payload, {- kind: "goal_cancelled",- cause: "system",- objective: "Example goal",- cancelledBy: "alice",- });- assert.equal((copied[0]!.payload as { text: string }).text, "hello");- assert.equal((copied.find((entry) => entry.seq === 5)!.payload as { text: string }).text, "final answer");- assert.equal((copied.find((entry) => entry.seq === 4)!.payload as { result: string }).result, "out\n");- assert.equal((copied.find((entry) => entry.seq === 3)!.payload as { callId: string }).callId, "c1");- assert.deepEqual(- copied.map((entry) => [entry.seq, entry.type, entry.createdAt]),- legacyEntries- .filter((entry) => !["soul", "text_start"].includes(entry.type))- .filter((entry) => (entry.payload as { kind?: string }).kind !== "goal")- .map((entry) => [entry.seq, entry.type, entry.createdAt]),- );- assert.deepEqual(- (await raw.query("SELECT seq, turn_id::text FROM items WHERE turn_id IS NOT NULL ORDER BY seq")).rows,- [- { seq: "0", turn_id: runId },- { seq: "10", turn_id: runId },- ],- );+ const store = createPostgresSessionStore(URL!);+ const copied = await store.getEntries(sessionId);+ assert.deepEqual(copied.at(-1)?.payload, {+ kind: "goal_cancelled",+ cause: "system",+ objective: "Example goal",+ cancelledBy: "alice",+ });+ assert.equal((copied[0]!.payload as { text: string }).text, "hello");+ assert.equal((copied.find((entry) => entry.seq === 5)!.payload as { text: string }).text, "final answer");+ assert.equal((copied.find((entry) => entry.seq === 4)!.payload as { result: string }).result, "out\n");+ assert.equal((copied.find((entry) => entry.seq === 3)!.payload as { callId: string }).callId, "c1");+ assert.deepEqual(+ copied.map((entry) => [entry.seq, entry.type, entry.createdAt]),+ legacyEntries+ .filter((entry) => !["soul", "text_start"].includes(entry.type))+ .filter((entry) => (entry.payload as { kind?: string }).kind !== "goal")+ .map((entry) => [entry.seq, entry.type, entry.createdAt]),+ );+ assert.deepEqual(+ (await raw.query("SELECT seq, turn_id::text FROM items WHERE turn_id IS NOT NULL ORDER BY seq")).rows,+ [+ { seq: "0", turn_id: runId },+ { seq: "10", turn_id: runId },+ ],+ );- const { lease } = await store.acquireLease(sessionId);- assert.ok(lease);- const next = await store.append(lease, {- type: "assistant",- payload: { text: "first answer after cutover zebracorn" },- scopeLabel: scope,- });- await store.releaseLease(lease);- assert.equal(next.seq, legacy.length);- assert.deepEqual(- (await store.searchEntries("alice", "zebracorn")).map((hit) => [hit.sessionId, hit.seq]),- [[sessionId, legacy.length]],- "search reads items",- );- assert.deepEqual(- (await store.searchEntries("alice", "final answer")).map((hit) => [hit.seq, hit.type]),- [[5, "assistant"]],- "copied history is searchable",- );+ const legacyTables = await raw.query(+ `SELECT name FROM unnest(ARRAY['session_entries', 'legacy_session_entries', 'session_entry_search',+ 'session_tape', 'legacy_tape_steps', 'legacy_turn_runs', 'legacy_tape_steps_progress']) AS name+ WHERE to_regclass(name) IS NOT NULL`,+ );+ assert.deepEqual(legacyTables.rows, [], "every legacy history table is dropped");+ const leftovers = await raw.query(+ `SELECT proname FROM pg_proc WHERE pronamespace = current_schema()::regnamespace+ AND proname IN ('entry_search_text', 'sync_item_search', 'items_copy_payload')+ UNION ALL SELECT tgname FROM pg_trigger WHERE tgrelid = 'items'::regclass AND tgname = 'session_entry_search'`,+ );+ assert.deepEqual(leftovers.rows, [], "legacy search and copy functions and triggers are dropped");- await assert.rejects(- raw.query("UPDATE items SET body = '{}' WHERE session_id = $1", [sessionId]),- /UPDATE on items is forbidden/,- );- await assert.rejects(- raw.query("DELETE FROM items WHERE session_id = $1", [sessionId]),- /only allowed through its session/,- );- await assert.rejects(raw.query("TRUNCATE items"), /TRUNCATE on items is forbidden/);- await raw.query(- "INSERT INTO blobs(tenant_id, sha256, size, mime, storage_key, created_at) VALUES (qm_tenant_id(), decode(repeat('00', 32), 'hex'), 1, 'text/plain', 'k', now())",- );- await assert.rejects(raw.query("UPDATE blobs SET size = 2"), /UPDATE on blobs is forbidden/);+ const { lease } = await store.acquireLease(sessionId);+ assert.ok(lease);+ const next = await store.append(lease, {+ type: "assistant",+ payload: { text: "first answer after cutover zebracorn" },+ scopeLabel: scope,+ });+ await store.releaseLease(lease);+ assert.equal(next.seq, legacy.length);+ assert.deepEqual(+ (await store.searchEntries("alice", "zebracorn")).map((hit) => [hit.sessionId, hit.seq]),+ [[sessionId, legacy.length]],+ "search reads items",+ );+ assert.deepEqual(+ (await store.searchEntries("alice", "final answer")).map((hit) => [hit.seq, hit.type]),+ [[5, "assistant"]],+ "copied history is searchable",+ );- await store.deleteSession(sessionId);- assert.equal(Number((await raw.query("SELECT count(*) FROM items")).rows[0].count), 0);- } finally {- await raw.end();- }-});+ await assert.rejects(+ raw.query("UPDATE items SET body = '{}' WHERE session_id = $1", [sessionId]),+ /UPDATE on items is forbidden/,+ );+ await assert.rejects(+ raw.query("DELETE FROM items WHERE session_id = $1", [sessionId]),+ /only allowed through its session/,+ );+ await assert.rejects(raw.query("TRUNCATE items"), /TRUNCATE on items is forbidden/);+ await raw.query(+ "INSERT INTO blobs(tenant_id, sha256, size, mime, storage_key, created_at) VALUES (qm_tenant_id(), decode(repeat('00', 32), 'hex'), 1, 'text/plain', 'k', now())",+ );+ await assert.rejects(raw.query("UPDATE blobs SET size = 2"), /UPDATE on blobs is forbidden/);++ await store.deleteSession(sessionId);+ assert.equal(Number((await raw.query("SELECT count(*) FROM items")).rows[0].count), 0);+ } finally {+ await raw.end();+ }+ },+);