Multiplayer agent harness for work.
History copy: items schema and resumable production backfill
Regan Bell2c6024dd37a8
diff --git a/package.json b/package.jsonindex 129d4fa85..7aaba0d2f 100644--- a/package.json+++ b/package.json@@ -48,7 +48,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.tsindex a9c7e92ab..8b5465e22 100644--- a/plugins/web-ui/test/timeline.test.ts+++ b/plugins/web-ui/test/timeline.test.ts@@ -586,3 +586,22 @@ test("live reasoning renders as a trailing thinking row until its persisted entrassert.equal(buildTimeline(otherBlock).length, 3, "a different block of the same step does not hide this one");assert.equal(buildTimeline({ ...work, status: "complete" }).length, 1, "finished work shows only durable rows");});++test("NULL legacy call IDs preserve the existing adjacent-result fallback", () => {+ for (const callId of [null, "known-call"]) {+ const work: WorkBlock = {+ status: "complete",+ activity: [+ act(10, "tool_call", { tool: "execute", command: "echo hello", callId }),+ act(11, "tool_result", { tool: "execute", result: "hello", code: 0, callId: null }),+ ],+ };+ const timeline = buildTimeline(work);+ assert.equal(timeline.length, 1);+ assert.equal(timeline[0].kind, "tool");+ if (timeline[0].kind !== "tool") throw new Error("expected paired tool row");+ assert.equal(timeline[0].row.call?.seq, 10);+ assert.equal(timeline[0].row.result?.seq, 11);+ assert.equal(toolRowKind(timeline[0].row, work.status), "ok");+ }+});diff --git a/scripts/ops/items-copy.ts b/scripts/ops/items-copy.tsnew file mode 100644index 000000000..8ab4033a0--- /dev/null+++ b/scripts/ops/items-copy.ts@@ -0,0 +1,529 @@+import pg from "pg";+import { pgMigrationChecksum } from "../../src/persistence/pg-pool.ts";+import { createHash } from "node:crypto";+import { parseArgs } from "node:util";+import { pathToFileURL } from "node:url";+import { sleep } from "../../src/util/async.ts";+import { ITEMS_TABLES_SQL, ITEM_INDEXES, itemIndexSql, itemsMigration } from "../../src/sessions/items-schema.ts";+import { itemsCopySelectedSql, itemsSelectedRowsSql, ITEMS_COPY_REPAIR_SQL } from "../../src/sessions/items-copy.ts";++const fingerprint = createHash("sha256")+ .update(JSON.stringify([ITEMS_TABLES_SQL, ITEM_INDEXES, ITEMS_COPY_REPAIR_SQL, itemsCopySelectedSql()]))+ .digest("hex");+const state = "items_copy_ops.state";+const dirty = "items_copy_ops.dirty";+const slices = "items_copy_ops.slices";+const SLICES_SQL = `CREATE TABLE IF NOT EXISTS ${slices}(slice int PRIMARY KEY,hi_session text,cursor_session text,cursor_seq int,+ scanned bigint NOT NULL DEFAULT 0,copied bigint NOT NULL DEFAULT 0,done boolean NOT NULL DEFAULT false)`;++export class ItemsCopyOperator {+ readonly client: pg.Client;+ readonly batch: number;+ constructor(client: pg.Client, batch = 2000) {+ this.client = client;+ this.batch = batch;+ if (!Number.isSafeInteger(batch) || batch < 1 || batch > 10000)+ throw new Error("batch must be between 1 and 10000");+ }++ async transaction<T>(+ run: () => Promise<T>,+ migrationLock: "exclusive" | "shared" | "none" = "exclusive",+ ): Promise<T> {+ await this.client.query(+ "BEGIN; SET LOCAL lock_timeout='2s'; SET LOCAL statement_timeout='120s'; SET LOCAL search_path=public",+ );+ try {+ const lock = `SELECT pg_try_advisory_xact_lock${migrationLock === "shared" ? "_shared" : ""}(hashtext('qm:schema-migrations')) AS acquired`;+ for (let attempt = 0; migrationLock !== "none" && !(await this.client.query(lock)).rows[0].acquired; attempt++) {+ if (attempt >= 300) throw new Error("schema migration is running; retry after it completes");+ await sleep(1000);+ }+ const result = await run();+ await this.client.query("COMMIT");+ return result;+ } catch (error) {+ try {+ await this.client.query("ROLLBACK");+ } catch (rollback) {+ throw new AggregateError([error, rollback], "items copy failed and rollback failed", { cause: rollback });+ }+ throw error;+ }+ }++ async prepare() {+ await this.transaction(async () => {+ if ((await this.client.query("SELECT to_regclass('public.items') AS existing")).rows[0].existing)+ throw new Error("items already exists; inspect it before preparing a new copy");+ await this.client.query(+ "LOCK TABLE public.session_entries, public.participants, public.directory_members, public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",+ );+ await this.client.query("CREATE SCHEMA items_copy_ops");+ for (const sql of ITEMS_TABLES_SQL) await this.client.query(sql);+ for (const sql of ITEMS_COPY_REPAIR_SQL) await this.client.query(sql);+ await this.client.query("CREATE EXTENSION IF NOT EXISTS btree_gin");+ await this.client.query(`CREATE TABLE ${state}(singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton),+ source_oid oid NOT NULL, target_oid oid NOT NULL, tenant_id uuid NOT NULL, fingerprint text NOT NULL,+ phase text NOT NULL, cursor_session text, cursor_seq int, scanned bigint NOT NULL DEFAULT 0,+ copied bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now())`);+ await this.client.query(+ `INSERT INTO ${state}(source_oid,target_oid,tenant_id,fingerprint,phase)+ VALUES('public.session_entries'::regclass,'public.items'::regclass,(SELECT id FROM public.qm_tenant),$1,'copy')`,+ [fingerprint],+ );+ await this.client.query(+ `CREATE TABLE ${dirty}(session_id text PRIMARY KEY,version bigint NOT NULL DEFAULT 1,copy_version bigint,cursor_seq int,touched_at timestamptz NOT NULL DEFAULT now())`,+ );+ await this.client.query(`CREATE INDEX dirty_oldest ON ${dirty}(touched_at,session_id)`);+ await this.client.query(SLICES_SQL);+ await this.client.query(`INSERT INTO ${slices}(slice) VALUES(0)`);+ await this.client.query(`CREATE FUNCTION items_copy_ops.capture() RETURNS trigger LANGUAGE plpgsql AS $capture$+ BEGIN+ IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'TRUNCATE is disabled during items copy'; END IF;+ IF TG_TABLE_NAME='directory_members' THEN+ IF TG_OP='UPDATE' AND (OLD.principal_id,OLD.display_name_lc) IS NOT DISTINCT FROM (NEW.principal_id,NEW.display_name_lc) THEN RETURN NULL; END IF;+ INSERT INTO ${dirty}(session_id)+ SELECT DISTINCT session_id FROM participants+ WHERE principal_id IN (CASE WHEN TG_OP<>'INSERT' THEN OLD.principal_id END,CASE WHEN TG_OP<>'DELETE' THEN NEW.principal_id END)+ ORDER BY session_id+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;+ ELSE+ IF TG_TABLE_NAME='participants' AND TG_OP='UPDATE' THEN+ IF (OLD.principal_id,OLD.session_id,OLD.valid_from,OLD.valid_to) IS NOT DISTINCT FROM (NEW.principal_id,NEW.session_id,NEW.valid_from,NEW.valid_to) THEN RETURN NULL; END IF;+ END IF;+ IF TG_OP<>'INSERT' THEN+ INSERT INTO ${dirty}(session_id) VALUES(OLD.session_id)+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;+ END IF;+ IF TG_OP<>'DELETE' THEN+ INSERT INTO ${dirty}(session_id) VALUES(NEW.session_id)+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;+ END IF;+ END IF;+ RETURN NULL;+ END $capture$`);+ for (const table of ["session_entries", "participants", "directory_members", "legacy_tape_steps"]) {+ await this.client.query(+ `CREATE TRIGGER items_copy_capture AFTER INSERT OR UPDATE OR DELETE ON public.${table} FOR EACH ROW EXECUTE FUNCTION items_copy_ops.capture()`,+ );+ await this.client.query(+ `CREATE TRIGGER items_copy_no_truncate BEFORE TRUNCATE ON public.${table} FOR EACH STATEMENT EXECUTE FUNCTION items_copy_ops.capture()`,+ );+ }+ });+ }++ async assertState(phases: string[], lock: "UPDATE" | "SHARE" = "UPDATE") {+ const {+ rows: [cutover],+ } = await this.client.query(+ "SELECT EXISTS(SELECT 1 FROM public.qm_schema_migrations WHERE id LIKE 'sessions/store/%items-cutover') AS done",+ );+ if (cutover.done) throw new Error("items cutover is already applied; the legacy copy operator must not run");+ const {+ rows: [s],+ } = await this.client.query(+ `SELECT *,source_oid='public.session_entries'::regclass AND target_oid='public.items'::regclass AND tenant_id=(SELECT id FROM public.qm_tenant) AS same FROM ${state} FOR ${lock}`,+ );+ if (!s?.same || s.fingerprint !== fingerprint || !phases.includes(s.phase))+ throw new Error("copy source, target, mapping or phase changed; inspect status");+ const {+ rows: [t],+ } = await this.client.query(+ "SELECT count(*)::int AS n FROM pg_trigger WHERE tgrelid IN ('public.session_entries'::regclass,'public.participants'::regclass,'public.directory_members'::regclass,'public.legacy_tape_steps'::regclass) AND tgname IN ('items_copy_capture','items_copy_no_truncate') AND tgenabled='O'",+ );+ if (t.n !== 8) throw new Error("items copy change capture is missing or disabled");+ await this.client.query(+ "CREATE TEMP TABLE IF NOT EXISTS items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq)) ON COMMIT DELETE ROWS",+ );+ return s;+ }++ async replaceSelected() {+ await this.client.query(+ "SELECT id FROM public.sessions WHERE id IN (SELECT session_id FROM pg_temp.items_copy_batch) ORDER BY id FOR KEY SHARE",+ );+ await this.client.query(+ "DELETE FROM public.items i USING pg_temp.items_copy_batch b WHERE i.tenant_id=(SELECT id FROM public.qm_tenant) AND (i.session_id,i.seq)=(b.session_id,b.seq)",+ );+ return (await this.client.query(itemsCopySelectedSql())).rows[0];+ }++ async split(workers: number) {+ if (!Number.isSafeInteger(workers) || workers < 1 || workers > 32)+ throw new Error("workers must be between 1 and 32");+ return this.transaction(async () => {+ const s = await this.assertState(["copy"]);+ if (!(await this.client.query(`SELECT to_regclass('${slices}') AS t`)).rows[0].t) {+ await this.client.query(SLICES_SQL);+ await this.client.query(`INSERT INTO ${slices}(slice,cursor_session,cursor_seq) VALUES(0,$1,$2)`, [+ s.cursor_session,+ s.cursor_seq,+ ]);+ }+ const { rows } = await this.client.query(`SELECT * FROM ${slices} ORDER BY slice FOR UPDATE`);+ const open = rows.filter((r) => !r.done);+ if (rows.length === workers || workers === 1) return rows.length;+ if (open.length !== 1)+ throw new Error(`copy is already split into ${rows.length} slices; rerun with --workers ${rows.length}`);+ const [from] = open;+ const {+ rows: [{ reltuples }],+ } = await this.client.query("SELECT reltuples FROM pg_class WHERE oid='public.session_entries'::regclass");+ const {+ rows: [{ bounds }],+ } = await this.client.query(+ `SELECT COALESCE(array_agg(DISTINCT b ORDER BY b),'{}') AS bounds FROM unnest((+ SELECT percentile_disc((SELECT array_agg(i::float8/$1) FROM generate_series(1,$1-1) i)) WITHIN GROUP (ORDER BY session_id)+ FROM public.session_entries TABLESAMPLE SYSTEM($2)+ WHERE ($3::text IS NULL OR session_id>$3) AND ($4::text IS NULL OR session_id<$4))) b+ WHERE b IS NOT NULL AND ($3::text IS NULL OR b>$3)`,+ [workers, Number(reltuples) > 1e6 ? 0.2 : 100, from.cursor_session, from.hi_session],+ );+ const his = [...bounds, from.hi_session];+ const base = Math.max(...rows.map((r) => r.slice)) + 1;+ await this.client.query(`UPDATE ${slices} SET hi_session=$2 WHERE slice=$1`, [from.slice, his[0]]);+ for (const [i, lo] of bounds.entries())+ await this.client.query(+ `INSERT INTO ${slices}(slice,hi_session,cursor_session,cursor_seq) VALUES($1,$2,$3,-1)`,+ [base + i, his[i + 1], lo],+ );+ return rows.length + bounds.length;+ });+ }++ async copyBatch(slice = 0) {+ return this.transaction(async () => {+ await this.assertState(["copy"], "SHARE");+ const {+ rows: [sl],+ } = await this.client.query(`SELECT * FROM ${slices} WHERE slice=$1 FOR UPDATE`, [slice]);+ if (!sl || sl.done) return 0;+ const params: unknown[] = [];+ const where: string[] = [];+ if (sl.cursor_session !== null)+ where.push(`(session_id,seq)>($${params.push(sl.cursor_session)},$${params.push(sl.cursor_seq)})`);+ if (sl.hi_session !== null) where.push(`session_id<$${params.push(sl.hi_session)}`);+ await this.client.query(+ `INSERT INTO pg_temp.items_copy_batch SELECT session_id,seq FROM public.session_entries+ ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY session_id,seq LIMIT $${params.push(this.batch)}`,+ params,+ );+ const {+ rows: [last],+ } = await this.client.query(+ "SELECT session_id,seq,(SELECT count(*)::int FROM pg_temp.items_copy_batch) AS n FROM pg_temp.items_copy_batch ORDER BY session_id DESC,seq DESC LIMIT 1",+ );+ if (!last) {+ await this.client.query(`UPDATE ${slices} SET done=true WHERE slice=$1`, [slice]);+ return 0;+ }+ const copied = await this.replaceSelected();+ await this.client.query(+ `UPDATE ${slices} SET cursor_session=$2,cursor_seq=$3,scanned=scanned+$4,copied=copied+$5 WHERE slice=$1`,+ [slice, last.session_id, last.seq, last.n, copied.rows],+ );+ return last.n as number;+ }, "shared");+ }++ async completeCopy() {+ return this.transaction(async () => {+ await this.assertState(["copy"]);+ const {+ rows: [open],+ } = await this.client.query(`SELECT count(*)::int AS n FROM ${slices} WHERE NOT done`);+ if (open.n) return false;+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);+ return true;+ });+ }++ async catchupBatch() {+ return this.transaction(async () => {+ await this.assertState(["catchup", "verified", "ready"]);+ return this.drainBatch();+ });+ }++ async drainBatch() {+ await this.client.query("TRUNCATE pg_temp.items_copy_batch");+ const {+ rows: [entry],+ } = await this.client.query(`SELECT * FROM ${dirty} ORDER BY touched_at,session_id LIMIT 1`);+ if (!entry) return 0;+ const version = entry.copy_version ?? entry.version;+ const params = [entry.session_id, entry.cursor_seq ?? -1, this.batch];+ await this.client.query(+ `INSERT INTO pg_temp.items_copy_batch+ WITH a AS (SELECT seq FROM public.session_entries WHERE session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3),+ b AS (SELECT seq::int FROM public.items WHERE tenant_id=(SELECT id FROM public.qm_tenant) AND session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3)+ SELECT $1,seq FROM (SELECT seq FROM a UNION SELECT seq FROM b) keys ORDER BY seq LIMIT $3`,+ params,+ );+ const {+ rows: [batch],+ } = await this.client.query("SELECT max(seq) AS last,count(*)::int AS n FROM pg_temp.items_copy_batch");+ if (batch.n) {+ await this.replaceSelected();+ await this.client.query(+ `UPDATE ${dirty} SET copy_version=$2,cursor_seq=$3,touched_at=clock_timestamp() WHERE session_id=$1`,+ [entry.session_id, version, batch.last],+ );+ } else {+ await this.client.query(`DELETE FROM ${dirty} WHERE session_id=$1 AND version=$2`, [entry.session_id, version]);+ await this.client.query(+ `UPDATE ${dirty} SET copy_version=NULL,cursor_seq=NULL,touched_at=clock_timestamp() WHERE session_id=$1`,+ [entry.session_id],+ );+ }+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);+ return batch.n || 1;+ }++ async indexes() {+ await this.transaction(async () => {+ await this.assertState(["catchup", "verified"]);+ }, "none");+ await this.client.query(+ "SET search_path=public; SET statement_timeout=0; SET lock_timeout='5s'; SET maintenance_work_mem='1GB'",+ );+ try {+ for (const index of ITEM_INDEXES) {+ const {+ rows: [existing],+ } = await this.client.query(+ "SELECT i.indisvalid, i.indrelid='public.items'::regclass AS ours FROM pg_index i WHERE indexrelid=to_regclass($1)",+ [`public.${index.name}`],+ );+ if (existing && !existing.ours) throw new Error(`index ${index.name} belongs to another table`);+ if (existing?.indisvalid) continue;+ if (existing) await this.client.query(`DROP INDEX public.${index.name}`);+ await this.client.query(itemIndexSql(index));+ }+ } finally {+ await this.client.query("RESET maintenance_work_mem");+ }+ }++ async finish(writersStopped: boolean) {+ if (!writersStopped)+ throw new Error("finish requires --writers-stopped; keep old writers stopped through the items cutover");+ await this.transaction(async () => {+ await this.client.query("SET LOCAL statement_timeout=0");+ await this.client.query(+ "LOCK TABLE public.session_entries,public.participants,public.directory_members,public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",+ );+ await this.assertState(["verified", "ready"]);+ const {+ rows: [leases],+ } = await this.client.query(+ "SELECT count(*)::int AS n FROM public.session_leases WHERE expires_at>(extract(epoch FROM clock_timestamp())*1000)::bigint",+ );+ if (leases.n) throw new Error("active session leases remain; drain writers before finishing the copy");+ while (await this.drainBatch()) continue;+ const {+ rows: [{ valid }],+ } = await this.client.query(+ `SELECT count(*)::int AS valid FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid+ WHERE i.indrelid='public.items'::regclass AND i.indisvalid AND c.relname=ANY($1)`,+ [ITEM_INDEXES.map((index) => index.name)],+ );+ if (valid !== ITEM_INDEXES.length)+ throw new Error("build the items indexes before stopping writers: run indexes");+ const checksum = pgMigrationChecksum(itemsMigration.statements);+ const {+ rows: [recorded],+ } = await this.client.query("SELECT checksum FROM public.qm_schema_migrations WHERE id=$1", [itemsMigration.id]);+ if (recorded && recorded.checksum !== checksum)+ throw new Error("items migration checksum differs from the staged copy");+ await this.client.query(+ "INSERT INTO public.qm_schema_migrations(id,checksum) VALUES($1,$2) ON CONFLICT(id) DO NOTHING",+ [itemsMigration.id, checksum],+ );+ await this.client.query(`UPDATE ${state} SET phase='ready',updated_at=now()`);+ });+ }++ async verify() {+ await this.transaction(async () => {+ await this.assertState(["catchup", "verified", "ready"]);+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);+ });+ for (const table of ["public.session_entries", "public.items"]) {+ let cursor: [string, number] | null = null;+ for (;;) {+ const last = await this.transaction(async () => {+ await this.assertState(["catchup", "verified", "ready"]);+ const params = cursor === null ? [this.batch] : [...cursor, this.batch];+ const where = [+ table === "public.items" ? "tenant_id=(SELECT id FROM public.qm_tenant)" : "",+ cursor === null ? "" : "(session_id,seq)>($1,$2)",+ ]+ .filter(Boolean)+ .join(" AND ");+ await this.client.query(+ `INSERT INTO pg_temp.items_copy_batch SELECT session_id,seq FROM ${table}+ ${where ? `WHERE ${where}` : ""} ORDER BY session_id,seq LIMIT $${params.length}`,+ params,+ );+ const {+ rows: [row],+ } = await this.client.query(+ "SELECT session_id,seq FROM pg_temp.items_copy_batch ORDER BY session_id DESC,seq DESC LIMIT 1",+ );+ if (!row) return null;+ const expected = itemsSelectedRowsSql();+ const {+ rows: [diff],+ } = await this.client.query(`WITH expected AS (${expected}),+ actual AS (SELECT i.* FROM public.items i JOIN pg_temp.items_copy_batch b USING(session_id,seq)+ WHERE tenant_id=(SELECT id FROM public.qm_tenant))+ SELECT count(*)::int AS n FROM (+ SELECT COALESCE(e.session_id,a.session_id) AS session_id,COALESCE(e.seq,a.seq) AS seq+ FROM expected e FULL JOIN actual a USING(tenant_id,session_id,seq)+ WHERE (to_jsonb(e)-'search'-'blob_sha') IS DISTINCT FROM (to_jsonb(a)-'search'-'blob_sha')+ ) d WHERE NOT EXISTS(SELECT 1 FROM ${dirty} q WHERE q.session_id=d.session_id)`);+ if (diff.n) throw new Error("items verification found differences without captured changes");+ return [row.session_id, row.seq] as [string, number];+ });+ if (!last) break;+ cursor = last;+ }+ }+ await this.transaction(async () => {+ await this.assertState(["catchup", "verified", "ready"]);+ const {+ rows: [duplicates],+ } = await this.client.query(`SELECT EXISTS(+ SELECT 1 FROM public.items WHERE call_id IS NOT NULL GROUP BY tenant_id,session_id,kind,call_id HAVING count(*)>1+ ) AS found`);+ if (duplicates.found) throw new Error("duplicate call IDs prevent the final items unique index");+ await this.client.query(`UPDATE ${state} SET phase='verified',updated_at=now()`);+ });+ }++ async cleanup() {+ await this.transaction(async () => {+ const {+ rows: [cutover],+ } = await this.client.query(+ "SELECT EXISTS(SELECT 1 FROM public.qm_schema_migrations WHERE id LIKE 'sessions/store/%items-cutover') AS done",+ );+ if (!cutover.done) throw new Error("cleanup requires the items cutover migration to be applied");+ for (const table of [+ "session_entries",+ "legacy_session_entries",+ "participants",+ "directory_members",+ "legacy_tape_steps",+ ]) {+ if ((await this.client.query("SELECT to_regclass($1) AS relation", [`public.${table}`])).rows[0].relation) {+ await this.client.query(`DROP TRIGGER IF EXISTS items_copy_capture ON public.${table}`);+ await this.client.query(`DROP TRIGGER IF EXISTS items_copy_no_truncate ON public.${table}`);+ }+ }+ await this.client.query("DROP FUNCTION IF EXISTS items_copy_ops.capture()");+ await this.client.query(`DROP TABLE IF EXISTS ${dirty},${state},${slices}`);+ await this.client.query("DROP SCHEMA IF EXISTS items_copy_ops");+ });+ }++ async report() {+ const { rows } = await this.client.query(`SELECT kind,count(*)::bigint AS rows,+ count(*) FILTER (WHERE author IS NULL)::bigint AS author_null,+ CASE WHEN kind IN ('UserMessage','SteeringMessage','ElicitationResponse') THEN+ round(100.0*count(*) FILTER (WHERE author IS NULL)/count(*),2) END AS author_null_percent+ FROM public.items GROUP BY kind ORDER BY kind`);+ return rows;+ }++ async status() {+ return (+ await this.client+ .query(`SELECT st.phase,st.scanned+sl.scanned AS scanned,st.copied+sl.copied AS initial_copied,sl.slices,sl.done,+ st.updated_at,(SELECT count(*) FROM ${dirty}) AS queued,pg_total_relation_size('public.items') AS bytes+ FROM ${state} st,(SELECT COALESCE(sum(scanned),0) AS scanned,COALESCE(sum(copied),0) AS copied,count(*)::int AS slices,+ count(*) FILTER (WHERE done)::int AS done FROM ${slices}) sl`)+ ).rows[0];+ }+}++async function main() {+ const { values, positionals } = parseArgs({+ allowPositionals: true,+ options: {+ batch: { type: "string", default: "2000" },+ workers: { type: "string", default: "1" },+ "writers-stopped": { type: "boolean", default: false },+ },+ });+ const action = positionals[0];+ if (+ !["prepare", "copy", "catchup", "indexes", "verify", "status", "report", "finish", "cleanup"].includes(action ?? "")+ )+ throw new Error(+ "usage: node scripts/ops/items-copy.ts prepare|copy|catchup|indexes|verify|status|report|finish|cleanup [--batch 2000] [--workers 1] [--writers-stopped]",+ );+ if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL must name the direct Postgres endpoint");+ const client = new pg.Client({ connectionString: process.env.DATABASE_URL, application_name: "qm-items-copy" });+ await client.connect();+ try {+ const op = new ItemsCopyOperator(client, Number(values.batch));+ if (action !== "status") {+ const {+ rows: [lock],+ } = await client.query("SELECT pg_try_advisory_lock(hashtext('qm:items-copy')) AS acquired");+ if (!lock.acquired) throw new Error("another items copy operator is running");+ }+ if (action === "cleanup") {+ await op.cleanup();+ console.log("items copy temporary state removed");+ return;+ }+ if (action === "prepare") await op.prepare();+ if (action === "indexes") await op.indexes();+ if (action === "verify") await op.verify();+ if (action === "finish") await op.finish(values["writers-stopped"]);+ if (action === "report") console.log(JSON.stringify(await op.report()));+ if (action === "copy") {+ await op.split(Number(values.workers));+ let failed: unknown = null;+ await Promise.all(+ (await client.query(`SELECT slice FROM ${slices} WHERE NOT done ORDER BY slice`)).rows.map(+ async ({ slice }) => {+ const worker = new pg.Client({+ connectionString: process.env.DATABASE_URL,+ application_name: "qm-items-copy",+ });+ await worker.connect();+ try {+ const w = new ItemsCopyOperator(worker, op.batch);+ for (let batches = 1; failed === null && (await w.copyBatch(slice)); batches++)+ if (batches % 20 === 0) console.log(JSON.stringify({ slice, ...(await w.status()) }));+ } catch (error) {+ failed ??= error;+ } finally {+ await worker.end();+ }+ },+ ),+ );+ if (failed !== null) throw failed;+ await op.completeCopy();+ }+ if (action === "catchup") {+ let batches = 0;+ while (await op.catchupBatch()) if (++batches % 20 === 0) console.log(JSON.stringify(await op.status()));+ }+ console.log(JSON.stringify(await op.status()));+ } finally {+ await client.end();+ }+}+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)+ main().catch((error) => {+ console.error(error);+ process.exitCode = 1;+ });diff --git a/src/directory/postgres-directory-store.ts b/src/directory/postgres-directory-store.tsindex 6b969a1aa..cf6e33f3f 100644--- a/src/directory/postgres-directory-store.ts+++ b/src/directory/postgres-directory-store.ts@@ -170,25 +170,27 @@ function dedupMemberships(rows: ChannelMembership[]): ChannelMembership[] {);}+export const DIRECTORY_MIGRATIONS = [+ {+ id: "directory/store/0001",+ expectedChecksum: "c47a45848f9d225ff601c9df13dc272349c7df7525e7e2417f33d68be7672cc1",+ statements: INITIAL_SCHEMA,+ },+ { id: "directory/store/0002", statements: SYNC_STAMP_SCHEMA },+ { id: "directory/store/0003", statements: EXTERNAL_ROSTER_SCHEMA },+ {+ id: "directory/store/0004",+ statements: ["ALTER TABLE directory_channels ADD COLUMN IF NOT EXISTS observed_at BIGINT"],+ },+ {+ id: "directory/store/0005",+ statements: ["ALTER TABLE directory_members ADD COLUMN IF NOT EXISTS avatar_url TEXT"],+ },+];+export function createPostgresDirectoryStore(connectionString: string): DirectoryStore {const orgId = configOrgId();- const { q, pool } = createPgPool(connectionString, [- {- id: "directory/store/0001",- expectedChecksum: "c47a45848f9d225ff601c9df13dc272349c7df7525e7e2417f33d68be7672cc1",- statements: INITIAL_SCHEMA,- },- { id: "directory/store/0002", statements: SYNC_STAMP_SCHEMA },- { id: "directory/store/0003", statements: EXTERNAL_ROSTER_SCHEMA },- {- id: "directory/store/0004",- statements: ["ALTER TABLE directory_channels ADD COLUMN IF NOT EXISTS observed_at BIGINT"],- },- {- id: "directory/store/0005",- statements: ["ALTER TABLE directory_members ADD COLUMN IF NOT EXISTS avatar_url TEXT"],- },- ]);+ const { q, pool } = createPgPool(connectionString, DIRECTORY_MIGRATIONS);async function pick<T>(query: string,diff --git a/src/sessions/items-copy.ts b/src/sessions/items-copy.tsnew file mode 100644index 000000000..ab6e34d4c--- /dev/null+++ b/src/sessions/items-copy.ts@@ -0,0 +1,237 @@+export const ITEMS_COPY_REPAIR_SQL = [+ String.raw`CREATE OR REPLACE FUNCTION items_copy_payload(t text) RETURNS jsonb+ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE AS $items_copy_payload$+ DECLARE r text;+ BEGIN+ IF t IS NULL OR strpos(t, E'\\u') = 0 THEN RETURN t::jsonb; END IF;+ IF pg_input_is_valid(t, 'jsonb') THEN RETURN t::jsonb; END IF;+ r := regexp_replace(t, '(?<!\\)((?:\\\\)*)\\u0000', '\1', 'g');+ IF NOT pg_input_is_valid(r, 'jsonb') THEN+ r := regexp_replace(regexp_replace(r,+ '(?<!\\)((?:\\\\)*)\\u[dD][89abAB][0-9a-fA-F]{2}(?!\\u[dD][c-fC-F][0-9a-fA-F]{2})', '\1\\ufffd', 'g'),+ '(?<!\\)((?:\\\\)*)(?<!\\u[dD][89abAB][0-9a-fA-F]{2})\\u[dD][c-fC-F][0-9a-fA-F]{2}', '\1\\ufffd', 'g');+ END IF;+ RETURN r::jsonb;+ END $items_copy_payload$`,+ `CREATE OR REPLACE FUNCTION items_copy_payload(j jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS 'SELECT j'`,+];++export interface ItemsCopyOptions {+ delta?: boolean;+ bucket?: { index: number; count: number };+}++const bucketPredicate = (col: string, bucket: { index: number; count: number }): string => {+ const { index, count } = bucket;+ if (!Number.isInteger(count) || count < 1 || !Number.isInteger(index) || index < 0 || index >= count)+ throw new Error(`items copy: bucket ${index} of ${count} is invalid`);+ return `((hashtextextended(${col}, 0) % ${count}) + ${count}) % ${count} = ${index}`;+};++const TENANT = `(SELECT id FROM qm_tenant)`;+const AUTHOR_KINDS_SQL = `('UserMessage', 'SteeringMessage', 'ElicitationResponse', 'Stop')`;++const str = (key: string): string =>+ `CASE WHEN jsonb_typeof(n.body -> '${key}') = 'string' THEN n.body ->> '${key}' END`;+const isType = (key: string, type: string): string => `jsonb_typeof(n.body -> '${key}') = '${type}'`;++const NORMALIZE = ` CROSS JOIN LATERAL (SELECT items_copy_payload(e.payload) AS j OFFSET 0) parsed+ CROSS JOIN LATERAL (+ SELECT CASE WHEN parsed.j IS NULL OR jsonb_typeof(parsed.j) = 'null' THEN '{}'::jsonb+ WHEN jsonb_typeof(parsed.j) = 'object' THEN parsed.j+ WHEN jsonb_typeof(parsed.j) = 'string' THEN jsonb_build_object('text', parsed.j)+ ELSE jsonb_build_object('value', parsed.j)+ END AS body+ OFFSET 0+ ) n+`;++function ambiguousCallsSql(selected: boolean): string {+ return `SELECT DISTINCT session_id, call_id FROM (+ SELECT e.session_id, e.type, n.body ->> 'callId' AS call_id+ FROM session_entries e+ ${NORMALIZE}+ WHERE e.type IN ('tool_call', 'tool_result')+ ${selected ? "AND e.session_id IN (SELECT DISTINCT session_id FROM pg_temp.items_copy_batch)" : ""}+ AND jsonb_typeof(n.body -> 'callId') = 'string'+ AND octet_length(n.body ->> 'callId') BETWEEN 1 AND 1024+ GROUP BY e.session_id, e.type, n.body ->> 'callId'+ HAVING count(*) > 1+ ) repeated`;+}++const CALL_IDS = ` LEFT JOIN ambiguous_calls ambiguous+ ON ambiguous.session_id=e.session_id AND ambiguous.call_id=n.body ->> 'callId'+`;++const KIND = ` CROSS JOIN LATERAL (+ SELECT (CASE e.type+ WHEN 'user' THEN CASE WHEN n.body @> '{"steered":true}' THEN 'SteeringMessage' ELSE 'UserMessage' END+ WHEN 'assistant' THEN 'AgentMessage'+ WHEN 'text' THEN 'AgentMessage'+ WHEN 'delivery' THEN 'AgentMessage'+ WHEN 'thinking' THEN 'Reasoning'+ WHEN 'tool_call' THEN 'ToolCall'+ WHEN 'tool_result' THEN 'ToolResult'+ WHEN 'approval_request' THEN 'Elicitation'+ WHEN 'approval_resolved' THEN 'ElicitationResponse'+ WHEN 'soul' THEN NULL+ WHEN 'text_start' THEN NULL+ WHEN 'system' THEN+ CASE n.body ->> 'kind'+ WHEN 'context_summary' THEN 'ContextCompaction'+ WHEN 'turn_failure' THEN 'Error'+ WHEN 'turn_stopped' THEN 'Stop'+ WHEN 'wake' THEN 'Wake'+ WHEN 'goal_cancelled' THEN 'Wake'+ WHEN 'message_revision' THEN 'Retraction'+ WHEN 'goal' THEN NULL+ WHEN 'memory_context' THEN NULL+ WHEN 'file_event' THEN NULL+ ELSE 'unmapped system entry kind=' || COALESCE(n.body ->> 'kind', '(none)') || ' at '+ || e.session_id || '/' || e.seq+ END+ ELSE 'unmapped entry type=' || e.type || ' at ' || e.session_id || '/' || e.seq+ END)::item_kind AS kind+ OFFSET 0+ ) k+`;++const HOIST = ` LEFT JOIN legacy_tape_steps l ON l.session_id = e.session_id AND l.seq = e.seq+ CROSS JOIN LATERAL (+ SELECT CASE WHEN k.kind IN ('UserMessage', 'SteeringMessage', 'AgentMessage', 'ContextCompaction') THEN 'text'+ WHEN k.kind = 'Reasoning' THEN 'thinking'+ WHEN k.kind = 'ToolResult' THEN 'result'+ WHEN k.kind = 'Error' THEN 'message'+ END AS text_key,+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult', 'Reasoning', 'AgentMessage') THEN+ COALESCE(CASE WHEN ${isType("step", "object")} THEN n.body -> 'step' END, l.step) END AS step_obj,+ pg_input_is_valid(n.body ->> 'runId', 'uuid') AS run_is_uuid+ ) t+ CROSS JOIN LATERAL (+ SELECT CASE WHEN jsonb_typeof(n.body -> t.text_key) = 'string' THEN t.text_key END AS text_key,+ CASE WHEN jsonb_typeof(t.step_obj -> 'index') = 'number' THEN (t.step_obj ->> 'index')::int END AS step,+ CASE WHEN t.step_obj IS NOT NULL+ THEN COALESCE(CASE WHEN ${isType("wire", "object")} THEN n.body -> 'wire' END, l.wire, '{}'::jsonb)+ || jsonb_build_object('message', t.step_obj - 'index')+ WHEN NOT n.body ? 'wire' AND l.wire IS NOT NULL THEN l.wire+ END AS wire+ ) s+ CROSS JOIN LATERAL (+ SELECT CASE WHEN k.kind IN ${AUTHOR_KINDS_SQL} AND n.body ->> 'sourceRole' IS DISTINCT FROM 'agent' THEN+ COALESCE(+ CASE WHEN NOT n.body @> '{"overheard":true}' THEN+ (SELECT min(p.principal_id) FROM participants p+ WHERE p.session_id = e.session_id AND p.valid_from <= e.created_at+ AND (p.valid_to IS NULL OR p.valid_to > e.created_at)+ HAVING count(*) = 1) END,+ CASE WHEN btrim(${str("name")}) <> '' THEN+ (SELECT min(p.principal_id) FROM participants p+ WHERE p.session_id = e.session_id+ AND EXISTS (SELECT 1 FROM directory_members d+ WHERE d.principal_id = p.principal_id+ AND d.display_name_lc = lower(btrim(n.body ->> 'name')))+ HAVING count(*) = 1) END)+ END AS author,+ CASE WHEN k.kind = 'Retraction' AND ${isType("ts", "string")} THEN+ (SELECT max(u.seq) FROM session_entries u+ WHERE u.session_id = e.session_id AND u.seq < e.seq AND u.type = 'user'+ AND items_copy_payload(u.payload) ->> 'ts' = n.body ->> 'ts') END AS target_seq+ OFFSET 0+ ) a+ CROSS JOIN LATERAL (+ SELECT n.body - array_remove(ARRAY[+ 'securityTainted',+ CASE WHEN e.type = 'assistant' OR k.kind = 'Error' THEN 'deliveryKey' END,+ s.text_key,+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN 'tool' END,+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN 'callId' END,+ CASE WHEN k.kind IN ('Elicitation', 'ElicitationResponse') THEN 'requestId' END,+ CASE WHEN k.kind = 'ToolResult' AND ${isType("isError", "boolean")} THEN 'isError' END,+ CASE WHEN k.kind = 'UserMessage' AND ${isType("hidden", "boolean")} THEN 'hidden' END,+ CASE WHEN k.kind = 'UserMessage' AND ${isType("ts", "string")} THEN 'ts' END,+ CASE WHEN e.type = 'user' THEN 'steered' END,+ CASE WHEN k.kind = 'ContextCompaction' AND ${isType("throughSeq", "number")} THEN 'throughSeq' END,+ CASE WHEN k.kind = 'ElicitationResponse' AND ${isType("approved", "boolean")} THEN 'approved' END,+ CASE WHEN e.type = 'system' AND n.body ->> 'kind' <> 'goal_cancelled' THEN 'kind' END,+ CASE WHEN k.kind = 'Wake' THEN 'cause' END,+ CASE WHEN t.run_is_uuid THEN 'runId' END,+ CASE WHEN t.step_obj IS NOT NULL THEN 'step' END,+ CASE WHEN s.wire IS NOT NULL THEN 'wire' END,+ CASE WHEN s.text_key = 'result' AND ${isType("stdout", "string")}+ AND strpos(n.body ->> 'result', n.body ->> 'stdout') > 0 THEN 'stdout' END,+ CASE WHEN s.text_key = 'result' AND ${isType("stderr", "string")}+ AND strpos(n.body ->> 'result', n.body ->> 'stderr') > 0 THEN 'stderr' END+ ]::text[], NULL)+ || CASE WHEN s.wire IS NOT NULL THEN jsonb_build_object('wire', s.wire) ELSE '{}'::jsonb END AS body+ ) x+`;++const COLUMNS = `tenant_id, session_id, seq, turn_id, kind, author, call_id, elicit_id, wake_cause, error_code,+ text, tool, step, variant, is_error, hidden, external_id, through_seq, target_seq, approved,+ body, created_at`;++const SELECT = `SELECT ${TENANT}, e.session_id, e.seq,+ CASE WHEN t.run_is_uuid THEN (n.body ->> 'runId')::uuid END,+ k.kind,+ a.author,+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') AND octet_length(${str("callId")}) <= 1024 AND ambiguous.call_id IS NULL+ THEN NULLIF(${str("callId")}, '') END,+ CASE WHEN k.kind IN ('Elicitation', 'ElicitationResponse') THEN+ COALESCE(NULLIF(${str("requestId")}, ''), 'missing-elicit:' || e.seq) END,+ CASE WHEN k.kind = 'Wake' THEN+ CASE WHEN n.body ? 'cause' THEN (n.body ->> 'cause')::wake_cause ELSE 'system'::wake_cause END END,+ CASE WHEN k.kind = 'Error' THEN 'turn_failure' END,+ NULLIF(n.body ->> s.text_key, ''),+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN NULLIF(${str("tool")}, '') END,+ s.step,+ CASE e.type WHEN 'text' THEN 'narration' WHEN 'delivery' THEN 'files_delivered' END,+ CASE WHEN k.kind = 'ToolResult' AND ${isType("isError", "boolean")} THEN (n.body -> 'isError')::boolean END,+ CASE WHEN k.kind = 'UserMessage' AND ${isType("hidden", "boolean")} THEN (n.body -> 'hidden')::boolean END,+ CASE WHEN k.kind = 'UserMessage' THEN NULLIF(${str("ts")}, '') END,+ CASE WHEN k.kind = 'ContextCompaction' AND ${isType("throughSeq", "number")} THEN (n.body ->> 'throughSeq')::numeric::bigint END,+ a.target_seq,+ CASE WHEN k.kind = 'ElicitationResponse' AND ${isType("approved", "boolean")} THEN (n.body -> 'approved')::boolean END,+ x.body,+ to_timestamp(e.created_at / 1000.0)`;++export function itemsCopySql(opts: ItemsCopyOptions = {}): string {+ const from = opts.delta+ ? ` FROM sessions sess+ CROSS JOIN LATERAL (+ SELECT COALESCE(max(i.seq), -1) AS seq FROM items i WHERE i.tenant_id = ${TENANT} AND i.session_id = sess.id+ ) copied+ CROSS JOIN LATERAL (+ SELECT * FROM session_entries d WHERE d.session_id = sess.id AND d.seq > copied.seq OFFSET 0+ ) e+`+ : ` FROM session_entries e+`;+ const bucket = opts.bucket ? `${bucketPredicate(opts.delta ? "sess.id" : "e.session_id", opts.bucket)}\n AND ` : "";+ return insertRows(`WITH ambiguous_calls AS MATERIALIZED (${ambiguousCallsSql(false)})+${SELECT}+${from}${NORMALIZE}${KIND}${CALL_IDS}${HOIST} WHERE ${bucket}k.kind IS NOT NULL`);+}++function insertRows(rows: string): string {+ return `WITH inserted AS (+ INSERT INTO items(${COLUMNS}) ${rows} RETURNING kind,author+ ) SELECT count(*)::int AS rows,+ count(*) FILTER (WHERE kind IN ${AUTHOR_KINDS_SQL})::int AS author_eligible,+ count(*) FILTER (WHERE kind IN ${AUTHOR_KINDS_SQL} AND author IS NULL)::int AS author_null+ FROM inserted`;+}++export function itemsSelectedRowsSql(): string {+ return `WITH ambiguous_calls AS MATERIALIZED (${ambiguousCallsSql(true)})+ SELECT * FROM (+ ${SELECT}+ FROM pg_temp.items_copy_batch b+ JOIN session_entries e ON (e.session_id,e.seq)=(b.session_id,b.seq)+ ${NORMALIZE}${KIND}${CALL_IDS}${HOIST} WHERE k.kind IS NOT NULL+ ) AS mapped(${COLUMNS})`;+}++export function itemsCopySelectedSql(): string {+ return insertRows(itemsSelectedRowsSql());+}diff --git a/src/sessions/items-schema.ts b/src/sessions/items-schema.tsnew file mode 100644index 000000000..5d6f07bdd--- /dev/null+++ b/src/sessions/items-schema.ts@@ -0,0 +1,147 @@+import { itemsCopySql, ITEMS_COPY_REPAIR_SQL } from "./items-copy.ts";++const ITEM_KINDS = [+ "UserMessage",+ "SteeringMessage",+ "AgentMessage",+ "Reasoning",+ "ToolCall",+ "ToolResult",+ "Elicitation",+ "ElicitationResponse",+ "Wake",+ "Error",+ "ContextCompaction",+ "Retraction",+ "Stop",+ "RuntimeChange",+] as const;++type ItemKind = (typeof ITEM_KINDS)[number];++const WAKE_CAUSES = ["cron", "webhook", "child_done", "system"] as const;++const sqlList = (values: readonly string[]): string => values.map((value) => `'${value}'`).join(", ");++const AUTHOR_KINDS = ["UserMessage", "SteeringMessage", "ElicitationResponse", "Stop", "RuntimeChange"] as const;++const HOISTED_COLUMN_KINDS: ReadonlyArray<[column: string, kinds: readonly ItemKind[]]> = [+ ["text", ["UserMessage", "SteeringMessage", "AgentMessage", "Reasoning", "ToolResult", "Error", "ContextCompaction"]],+ ["tool", ["ToolCall", "ToolResult"]],+ ["step", ["ToolCall", "ToolResult", "Reasoning", "AgentMessage"]],+ ["variant", ["AgentMessage"]],+ ["is_error", ["ToolResult"]],+ ["hidden", ["UserMessage"]],+ ["external_id", ["UserMessage"]],+ ["through_seq", ["ContextCompaction"]],+ ["target_seq", ["Retraction"]],+ ["approved", ["ElicitationResponse"]],+ ["blob_sha", ["ToolResult"]],+];++const hoistedChecks = HOISTED_COLUMN_KINDS.map(+ ([column, kinds]) => `CONSTRAINT items_${column}_kind CHECK (${column} IS NULL OR kind IN (${sqlList(kinds)}))`,+).join(",\n ");++export const ITEMS_TABLES_SQL: readonly string[] = [+ `CREATE TABLE IF NOT EXISTS qm_tenant(id UUID PRIMARY KEY)`,+ `CREATE UNIQUE INDEX IF NOT EXISTS qm_tenant_singleton ON qm_tenant((true))`,+ `INSERT INTO qm_tenant(id) SELECT gen_random_uuid() WHERE NOT EXISTS (SELECT 1 FROM qm_tenant)`,+ `CREATE TABLE IF NOT EXISTS qm_settings(+ key TEXT PRIMARY KEY,+ value JSONB NOT NULL,+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()+ )`,+ `DO $item_kind$ BEGIN+ CREATE TYPE item_kind AS ENUM (${sqlList(ITEM_KINDS)});+ EXCEPTION WHEN duplicate_object THEN NULL; END $item_kind$`,+ `DO $wake_cause$ BEGIN+ CREATE TYPE wake_cause AS ENUM (${sqlList(WAKE_CAUSES)});+ EXCEPTION WHEN duplicate_object THEN NULL; END $wake_cause$`,+ `CREATE TABLE IF NOT EXISTS blobs(+ tenant_id UUID NOT NULL,+ sha256 BYTEA NOT NULL CHECK (length(sha256) = 32),+ size BIGINT NOT NULL CHECK (size >= 0),+ mime TEXT,+ storage_key TEXT NOT NULL CHECK (storage_key <> ''),+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),+ expires_at TIMESTAMPTZ,+ PRIMARY KEY (tenant_id, sha256)+ )`,+ `CREATE TABLE IF NOT EXISTS items(+ tenant_id UUID NOT NULL,+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,+ seq BIGINT NOT NULL CHECK (seq >= 0),+ turn_id UUID,+ kind item_kind NOT NULL,+ author TEXT CHECK (author <> ''),+ call_id TEXT CHECK (call_id <> ''),+ elicit_id TEXT CHECK (elicit_id <> ''),+ wake_cause wake_cause,+ error_code TEXT CHECK (error_code <> ''),+ text TEXT CHECK (text <> ''),+ tool TEXT CHECK (tool <> ''),+ step INT CHECK (step >= 0),+ variant TEXT CHECK (variant IN ('narration', 'files_delivered')),+ is_error BOOLEAN,+ hidden BOOLEAN,+ external_id TEXT CHECK (external_id <> ''),+ through_seq BIGINT,+ target_seq BIGINT,+ approved BOOLEAN,+ blob_sha BYTEA,+ search TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', coalesce(text, ''))) STORED,+ body JSONB NOT NULL CHECK (jsonb_typeof(body) = 'object'),+ created_at TIMESTAMPTZ NOT NULL,+ PRIMARY KEY (tenant_id, session_id, seq),+ CONSTRAINT items_call_id_kind CHECK (call_id IS NULL OR kind IN ('ToolCall', 'ToolResult')),+ CONSTRAINT items_elicit_id_kind CHECK ((elicit_id IS NOT NULL) = (kind IN ('Elicitation', 'ElicitationResponse'))),+ CONSTRAINT items_wake_cause_kind CHECK ((wake_cause IS NOT NULL) = (kind = 'Wake')),+ CONSTRAINT items_error_code_kind CHECK ((error_code IS NOT NULL) = (kind = 'Error')),+ CONSTRAINT items_author_kind CHECK (author IS NULL OR kind IN (${sqlList(AUTHOR_KINDS)})),+ ${hoistedChecks},+ CONSTRAINT items_blob FOREIGN KEY (tenant_id, blob_sha) REFERENCES blobs(tenant_id, sha256)+ )`,+];++export interface ItemIndex {+ name: string;+ unique?: boolean;+ on: string;+}++export const ITEM_INDEXES: readonly ItemIndex[] = [+ {+ name: "items_one_call_per_kind",+ unique: true,+ on: `items(tenant_id, session_id, kind, call_id) WHERE call_id IS NOT NULL`,+ },+ { name: "items_by_turn", on: `items(tenant_id, turn_id, seq) WHERE turn_id IS NOT NULL` },+ { name: "items_by_elicit", on: `items(tenant_id, session_id, elicit_id) WHERE elicit_id IS NOT NULL` },+ {+ name: "items_by_external_id",+ on: `items(tenant_id, session_id, external_id) WHERE external_id IS NOT NULL`,+ },+ { name: "items_session_kind_created", on: `items(tenant_id, session_id, kind, created_at)` },+ { name: "items_search", on: `items USING gin (tenant_id, session_id, search)` },+];++export function itemIndexSql(index: ItemIndex, opts: { concurrently?: boolean } = {}): string {+ return `CREATE ${index.unique ? "UNIQUE " : ""}INDEX ${opts.concurrently ? "CONCURRENTLY " : ""}IF NOT EXISTS ${index.name} ON ${index.on}`;+}++export const itemsMigration = {+ id: "sessions/store/0035-items",+ statements: [+ `DO $operator$ BEGIN+ IF to_regclass('items_copy_ops.state') IS NOT NULL THEN+ RAISE EXCEPTION 'An items copy is staged; complete the coordinated cutover before applying this migration';+ END IF;+ END $operator$`,+ ...ITEMS_TABLES_SQL,+ ...ITEMS_COPY_REPAIR_SQL,+ `SET LOCAL statement_timeout = 0`,+ itemsCopySql({ delta: true }),+ ...ITEM_INDEXES.map((index) => itemIndexSql(index)),+ ],+};diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.tsindex 69c578547..695dc7141 100644--- a/src/sessions/postgres-session-store.ts+++ b/src/sessions/postgres-session-store.ts@@ -1,3 +1,4 @@+import { DIRECTORY_MIGRATIONS } from "../directory/postgres-directory-store.ts";import { randomUUID } from "node:crypto";import {createPgPool,@@ -40,6 +41,7 @@ import type {StoreOptions,} from "./session-store.ts";import { tsPrefixQuery } from "./entry-search.ts";+import { itemsMigration } from "./items-schema.ts";import {DROP_SESSION_TAPE_SQL,LEGACY_TAPE_STEPS_SIZE_GUARD_SQL,@@ -345,6 +347,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Storeconst { pool, q } = createPgPool(connectionString,[+ ...DIRECTORY_MIGRATIONS,{id: "sessions/store/0001",expectedChecksum: "cf56c9f6488806677a229698193ef6dd7cb74300c343bdb1930c17702c5ccf2f",@@ -880,6 +883,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Storeid: "sessions/store/0033-client-tools",statements: [`ALTER TABLE sessions ADD COLUMN IF NOT EXISTS client_tools JSONB`],},+ itemsMigration,],[{diff --git a/test/loop-fire.test.ts b/test/loop-fire.test.tsindex 7bdcb6b72..0256c91d9 100644--- a/test/loop-fire.test.ts+++ b/test/loop-fire.test.ts@@ -1013,16 +1013,17 @@ test("a failed followup rejects so the composer can retain uploaded attachments"});test("triage groups a flood read-only so only the representative is worked, with the rest as evidence", async () => {- const s = service((req) => {+ const s = service(async (req) => {const text = req.text ?? "";if (text.startsWith("[Loop triage]")) {- const ids = [...text.matchAll(/"id":"([^"]+)"/g)].map((match) => match[1]!);+ const open = await s.items.byLoop(loop.id);+ const representative = open.find((item) => item.sourceKey === "SENTRY-1")!;return `\`\`\`json\n${JSON.stringify({- items: ids.map((id, index) => ({- id,- priority: index === 0 ? "urgent" : "low",+ items: open.map((item) => ({+ id: item.id,+ priority: item.id === representative.id ? "urgent" : "low",reason: "checkout is down",- ...(index > 0 ? { groupWith: ids[0] } : {}),+ ...(item.id !== representative.id ? { groupWith: representative.id } : {}),})),})}\n\`\`\``;}diff --git a/test/postgres-items-copy.test.ts b/test/postgres-items-copy.test.tsnew file mode 100644index 000000000..4cdfd5294--- /dev/null+++ b/test/postgres-items-copy.test.ts@@ -0,0 +1,575 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import { randomUUID } from "node:crypto";+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";+import { itemsCopySql, itemsCopySelectedSql, itemsSelectedRowsSql } from "../src/sessions/items-copy.ts";+import { ITEM_INDEXES } from "../src/sessions/items-schema.ts";+import { scopeId } from "../src/types.ts";++type Pool = import("pg").Pool;++const URL = process.env.DATABASE_URL;+const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the items copy test";+const T0 = 1_700_000_000_000;++const ITEMS_MIGRATION = "sessions/store/0035-items";++async function withSchema(+ fn: (raw: Pool, url: string) => Promise<void>,+ opts: { beforeItems?: (raw: Pool) => Promise<void> } = {},+): Promise<void> {+ const pg = (await import("pg")).default;+ const admin = new pg.Pool({ connectionString: URL });+ const schema = `items_copy_${randomUUID().replaceAll("-", "")}`;+ await admin.query(`CREATE SCHEMA ${schema}`);+ const url = new globalThis.URL(URL!);+ url.searchParams.set("options", `-c search_path=${schema}`);+ const raw = new pg.Pool({ connectionString: url.toString() });+ try {+ createPostgresSessionStore(url.toString());+ createPostgresDirectoryStore(url.toString());+ const migrations = registeredPgMigrations(url.toString());+ if (opts.beforeItems) {+ await applyPgMigrations(+ raw,+ migrations.filter((migration) => migration.id !== ITEMS_MIGRATION),+ );+ await opts.beforeItems(raw);+ }+ await applyPgMigrations(raw, migrations);+ await fn(raw, url.toString());+ } finally {+ await raw.end();+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);+ await admin.end();+ }+}++const scope = scopeId("personal", "alice");++async function seed(raw: Pool, id: string, entries: Array<[string, unknown]>, from = 0) {+ await raw.query(+ "INSERT INTO sessions(id, type, scope_id, thread_ref, created_at) VALUES ($1, 'dm', $2, $3, 1) ON CONFLICT DO NOTHING",+ [id, scope, `web:${id}`],+ );+ for (const [i, [type, payload]] of entries.entries())+ await raw.query(+ "INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at) VALUES ($1, $2, NULL, $3, $4, $5, $6)",+ [id, from + i, type, JSON.stringify(payload), scope, T0 + from + i],+ );+}++async function copyAll(raw: Pool, buckets = 4, delta = false): Promise<number> {+ let total = 0;+ for (let index = 0; index < buckets; index++)+ total += (await raw.query(itemsCopySql({ delta, bucket: { index, count: buckets } }))).rows[0].rows;+ return total;+}++const items = async (raw: Pool, sessionId: string) =>+ (+ await raw.query(+ `SELECT seq::int, turn_id, kind, author, call_id, elicit_id, wake_cause, error_code, text, tool, step,+ variant, is_error, hidden, external_id, through_seq::int, target_seq::int, approved, body,+ (extract(epoch FROM created_at) * 1000)::bigint::text AS ms+ FROM items WHERE session_id = $1 ORDER BY seq`,+ [sessionId],+ )+ ).rows as Array<Record<string, unknown> & { seq: number; body: Record<string, unknown> }>;++test("items copy maps every legacy type, hoists fields into columns and strips them from body", { skip }, async () => {+ await withSchema(async (raw) => {+ const run = randomUUID();+ await seed(raw, "s1", [+ ["soul", { text: "persona" }],+ ["user", { text: "hi", runId: run, securityTainted: true, ts: "1.1", hidden: true, name: "Al" }],+ ["user", { text: "also", steered: true, ts: "1.5" }],+ ["text_start", {}],+ ["text", { text: "narrating" }],+ ["thinking", { thinking: "hmm", thinkingSignature: "sig" }],+ ["tool_call", { tool: "execute", callId: "c1", args: { cmd: "ls" } }],+ [+ "tool_result",+ {+ tool: "execute",+ callId: "c1",+ result: "out: hello\nerr: boom",+ stdout: "hello",+ stderr: "nope",+ isError: true,+ },+ ],+ ["tool_call", { tool: "execute" }],+ ["assistant", { text: "done", deliveryKey: "k1", runId: "not-a-uuid", via: "cron" }],+ ["delivery", { files: ["a.txt"] }],+ ["approval_request", { requestId: "r1", command: "rm" }],+ ["approval_resolved", { approved: true }],+ ["system", { kind: "context_summary", text: "summary", throughSeq: 3 }],+ ["system", { kind: "turn_failure", message: "failed", deliveryKey: "k2", runId: run }],+ ["system", { kind: "wake", text: "follow-up" }],+ ["system", { kind: "wake", cause: "cron", text: "fire" }],+ ["system", { kind: "message_revision", action: "deleted", ts: "1.1" }],+ ["system", { kind: "goal", goal: {} }],+ ["system", { kind: "memory_context", snapshot: {} }],+ ["system", { kind: "file_event", direction: "in" }],+ ["user", "just a string"],+ ["user", { type: "x", $type: "y", $blobs: [], text: "t" }],+ ["assistant", 5],+ ["tool_result", { callId: "", result: { ok: true }, stdout: "kept" }],+ ["user", { text: "" }],+ ]);+ await raw.query(+ `INSERT INTO legacy_tape_steps(session_id, seq, step, wire)+ VALUES ('s1', 6, '{"id":"tape:1","model":"m","index":2}', '{"x":1}'), ('s1', 5, '{"id":"tape:1","index":0}', NULL)`,+ );++ assert.equal(await copyAll(raw), 21);+ const rows = await items(raw, "s1");+ const at = (seq: number) => rows.find((row) => row.seq === seq)!;+ assert.deepEqual(+ rows.map((row) => [row.seq, row.kind]),+ [+ [1, "UserMessage"],+ [2, "SteeringMessage"],+ [4, "AgentMessage"],+ [5, "Reasoning"],+ [6, "ToolCall"],+ [7, "ToolResult"],+ [8, "ToolCall"],+ [9, "AgentMessage"],+ [10, "AgentMessage"],+ [11, "Elicitation"],+ [12, "ElicitationResponse"],+ [13, "ContextCompaction"],+ [14, "Error"],+ [15, "Wake"],+ [16, "Wake"],+ [17, "Retraction"],+ [21, "UserMessage"],+ [22, "UserMessage"],+ [23, "AgentMessage"],+ [24, "ToolResult"],+ [25, "UserMessage"],+ ],+ );+ assert.equal(at(1).ms, String(T0 + 1));+ assert.deepEqual(+ [at(1).turn_id, at(1).text, at(1).hidden, at(1).external_id, at(1).body],+ [run, "hi", true, "1.1", { name: "Al" }],+ );+ assert.deepEqual([at(2).text, at(2).external_id, at(2).body], ["also", null, { ts: "1.5" }]);+ assert.deepEqual([at(4).text, at(4).variant, at(4).body], ["narrating", "narration", {}]);+ assert.deepEqual(+ [at(5).text, at(5).step, at(5).body],+ ["hmm", 0, { thinkingSignature: "sig", wire: { message: { id: "tape:1" } } }],+ );+ assert.deepEqual(+ [at(6).call_id, at(6).tool, at(6).step, at(6).body],+ ["c1", "execute", 2, { args: { cmd: "ls" }, wire: { x: 1, message: { id: "tape:1", model: "m" } } }],+ );+ assert.deepEqual(+ [at(7).call_id, at(7).tool, at(7).text, at(7).is_error, at(7).body],+ ["c1", "execute", "out: hello\nerr: boom", true, { stderr: "nope" }],+ );+ assert.equal(at(8).call_id, null);+ assert.deepEqual(+ [at(9).turn_id, at(9).text, at(9).variant, at(9).body],+ [null, "done", null, { runId: "not-a-uuid", via: "cron" }],+ );+ assert.deepEqual([at(10).variant, at(10).text, at(10).body], ["files_delivered", null, { files: ["a.txt"] }]);+ assert.deepEqual([at(11).elicit_id, at(11).body], ["r1", { command: "rm" }]);+ assert.deepEqual([at(12).elicit_id, at(12).approved, at(12).body], ["missing-elicit:12", true, {}]);+ assert.deepEqual([at(13).text, at(13).through_seq, at(13).body], ["summary", 3, {}]);+ assert.deepEqual(+ [at(14).error_code, at(14).text, at(14).turn_id, at(14).body],+ ["turn_failure", "failed", run, {}],+ );+ assert.deepEqual([at(15).wake_cause, at(15).body], ["system", { text: "follow-up" }]);+ assert.deepEqual([at(16).wake_cause, at(16).body], ["cron", { text: "fire" }]);+ assert.deepEqual([at(17).target_seq, at(17).body], [1, { action: "deleted", ts: "1.1" }]);+ assert.deepEqual([at(21).text, at(21).body], ["just a string", {}]);+ assert.deepEqual([at(22).text, at(22).body], ["t", { type: "x", $type: "y", $blobs: [] }]);+ assert.deepEqual(at(23).body, { value: 5 });+ assert.deepEqual(+ [at(24).call_id, at(24).text, at(24).body],+ [null, null, { result: { ok: true }, stdout: "kept" }],+ );+ assert.deepEqual([at(25).text, at(25).body], [null, {}]);+ });+});++test("items copy deduces legacy authors from participants and directory names", { skip }, async () => {+ await withSchema(async (raw) => {+ const join = (session: string, principal: string, from: number, to: number | null) =>+ raw.query("INSERT INTO participants(session_id, principal_id, valid_from, valid_to) VALUES ($1, $2, $3, $4)", [+ session,+ principal,+ from,+ to,+ ]);+ await raw.query(+ `INSERT INTO directory_members(org_id, principal_id, display_name, display_name_lc, type)+ VALUES ('o', 'u:ann', 'Ann Lee', 'ann lee', 'internal'), ('o', 'u:bob', 'Bob', 'bob', 'internal')`,+ );+ await seed(raw, "solo", [+ ["user", { text: "before ann joined" }],+ ["user", { text: "ann alone" }],+ ["user", { text: "both here", name: "bob" }],+ ["user", { text: "both here, nameless" }],+ ["user", { text: "overheard", overheard: true, ts: "9.1", name: "Ann Lee" }],+ ["user", { text: "agent echo", overheard: true, ts: "9.2", sourceRole: "agent", name: "Ann Lee" }],+ ["approval_resolved", { requestId: "r", approved: false }],+ ]);+ await join("solo", "u:ann", T0 + 1, null);+ await join("solo", "u:bob", T0 + 2, T0 + 6);+ const counts = (await raw.query(itemsCopySql())).rows[0];+ assert.deepEqual(counts, { rows: 7, author_eligible: 7, author_null: 3 });+ assert.deepEqual(+ (await items(raw, "solo")).map((row) => [row.seq, row.author]),+ [+ [0, null],+ [1, "u:ann"],+ [2, "u:bob"],+ [3, null],+ [4, "u:ann"],+ [5, null],+ [6, "u:ann"],+ ],+ );+ });+});++test("items CHECKs tie each hoisted column to its kinds and search indexes the text", { skip }, async () => {+ await withSchema(async (raw) => {+ await seed(raw, "c", []);+ const insert = (cols: string, values: string) =>+ raw.query(+ `INSERT INTO items(tenant_id, session_id, seq, body, created_at, ${cols})+ SELECT id, 'c', (SELECT count(*) FROM items), '{}', now(), ${values} FROM qm_tenant`,+ );+ await insert("kind, text", "'UserMessage', 'the quick brown fox'");+ await insert("kind, call_id, tool, is_error", "'ToolResult', 'c1', 'execute', false");+ for (const [cols, values, constraint] of [+ ["kind, tool", "'UserMessage', 'execute'", /items_tool_kind/],+ ["kind, variant", "'AgentMessage', 'reply'", /variant_check/],+ ["kind, text", "'UserMessage', ''", /text_check/],+ ["kind, hidden", "'SteeringMessage', true", /items_hidden_kind/],+ ["kind, external_id", "'AgentMessage', '1.1'", /items_external_id_kind/],+ ["kind, through_seq", "'Retraction', 1", /items_through_seq_kind/],+ ["kind, target_seq", "'ContextCompaction', 1", /items_target_seq_kind/],+ ["kind, approved", "'Elicitation', true", /items_approved_kind/],+ ["kind, author", "'AgentMessage', 'u:ann'", /items_author_kind/],+ ["kind, call_id, blob_sha", "'ToolResult', 'c2', sha256('x')", /items_blob/],+ ] as const)+ await assert.rejects(insert(cols, values), constraint);+ await insert("kind, author", "'Stop', 'u:ann'");+ await insert("kind, author", "'RuntimeChange', 'u:ann'");+ const hits = await raw.query(+ "SELECT seq::int FROM items WHERE session_id = 'c' AND search @@ to_tsquery('simple', 'brown & fox')",+ );+ assert.deepEqual(hits.rows, [{ seq: 0 }]);+ });+});++test("items copy fails loudly on an unmapped entry type or system kind", { skip }, async () => {+ await withSchema(async (raw) => {+ await seed(raw, "ok", [["user", { text: "fine" }]]);+ await seed(raw, "bad", [["system", { kind: "mystery" }]]);+ await assert.rejects(copyAll(raw, 1), /unmapped system entry kind=mystery at bad\/0/);+ await raw.query("DELETE FROM session_entries WHERE session_id = 'bad'");+ await seed(raw, "bad2", [["weird", { text: "?" }]]);+ await assert.rejects(copyAll(raw, 1), /unmapped entry type=weird at bad2\/0/);+ assert.equal((await raw.query("SELECT count(*)::int AS n FROM items")).rows[0].n, 0);+ });+});++test("bucketed bulk copy partitions sessions and delta copies only new entries", { skip }, async () => {+ await withSchema(async (raw) => {+ for (let i = 0; i < 12; i++)+ await seed(raw, `s${i}`, [+ ["user", { text: `q${i}` }],+ ["assistant", { text: `a${i}` }],+ ]);+ assert.equal(await copyAll(raw), 24);+ assert.equal(await copyAll(raw, 4, true), 0);+ await seed(+ raw,+ "s3",+ [+ ["user", { text: "later" }],+ ["system", { kind: "goal" }],+ ["thinking", { thinking: "x" }],+ ],+ 2,+ );+ await seed(raw, "fresh", [["user", { text: "new session" }]]);+ assert.equal(await copyAll(raw, 1, true), 3);+ assert.deepEqual(+ (await items(raw, "s3")).map((row) => [row.seq, row.kind]),+ [+ [0, "UserMessage"],+ [1, "AgentMessage"],+ [2, "UserMessage"],+ [4, "Reasoning"],+ ],+ );+ assert.equal(await copyAll(raw, 1, true), 0);+ assert.throws(() => itemsCopySql({ bucket: { index: 4, count: 4 } }), /bucket 4 of 4 is invalid/);+ });+});++test("the startup migration copies existing history and builds the indexes", { skip }, async () => {+ await withSchema(+ async (raw) => {+ assert.deepEqual(+ (await items(raw, "old")).map((row) => [row.seq, row.kind, row.text]),+ [+ [0, "UserMessage", "before items existed"],+ [1, "ToolCall", null],+ ],+ );+ const built = await raw.query(+ `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid+ WHERE i.indrelid = 'items'::regclass AND i.indisvalid AND c.relname = ANY($1)`,+ [ITEM_INDEXES.map((index) => index.name)],+ );+ assert.equal(built.rowCount, ITEM_INDEXES.length);+ },+ {+ beforeItems: (raw) =>+ seed(raw, "old", [+ ["user", { text: "before items existed" }],+ ["tool_call", { tool: "deliver", callId: "c1" }],+ ]),+ },+ );+});++test(+ "items copy repairs unsupported JSON escapes, preserves literals and rejects other broken payloads",+ { skip },+ async () => {+ await withSchema(async (raw) => {+ await seed(raw, "repair", []);+ const payloads = [+ String.raw`{"text":"nul\u0000end"}`,+ String.raw`{"text":"high\ud800end"}`,+ String.raw`{"text":"low\udc00end"}`,+ String.raw`{"text":"pair\ud83d\ude00"}`,+ String.raw`{"text":"literal\\u0000"}`,+ null,+ "7",+ "true",+ ];+ for (const [seq, payload] of payloads.entries())+ await raw.query(+ "INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES('repair',$1,'user',$2,'personal:test',1)",+ [seq, payload],+ );+ assert.equal(await copyAll(raw, 1), 8);+ const result = await items(raw, "repair");+ assert.deepEqual(+ result.slice(0, 5).map((r) => r.text),+ ["nulend", "high�end", "low�end", "pair😀", String.raw`literal\u0000`],+ );+ assert.deepEqual(+ result.slice(5).map((r) => r.body),+ [{}, { value: 7 }, { value: true }],+ );+ await raw.query("UPDATE session_entries SET payload='not json' WHERE seq=0");+ await raw.query("DELETE FROM items");+ await assert.rejects(copyAll(raw, 1), /invalid input syntax/);+ assert.equal((await raw.query("SELECT count(*)::int AS n FROM items")).rows[0].n, 0);+ });+ },+);++test("selected-key copying uses the same mapping on TEXT and JSONB without copying other rows", { skip }, async () => {+ await withSchema(async (raw) => {+ await seed(raw, "selected", [+ ["user", { text: "one" }],+ ["assistant", { text: "two" }],+ ["soul", {}],+ ]);+ const client = await raw.connect();+ try {+ await client.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");+ await client.query("INSERT INTO items_copy_batch VALUES('selected',1),('selected',2)");+ assert.equal((await client.query(itemsCopySelectedSql())).rows[0].rows, 1);+ assert.deepEqual(+ (await items(raw, "selected")).map((r) => r.text),+ ["two"],+ );+ const expected = (await client.query(itemsSelectedRowsSql())).rows[0];+ assert.equal(expected.kind, "AgentMessage");+ assert.equal(expected.text, "two");+ await client.query(+ "CREATE TEMP TABLE source_rows AS SELECT session_id,seq,type,payload,created_at FROM session_entries",+ );+ await client.query("CREATE TEMP TABLE session_entries AS SELECT * FROM source_rows WITH NO DATA");+ await client.query("ALTER TABLE pg_temp.session_entries ALTER COLUMN payload TYPE jsonb USING payload::jsonb");+ await client.query(+ "INSERT INTO pg_temp.session_entries SELECT session_id,seq,type,payload::jsonb,created_at FROM source_rows",+ );+ await client.query("DELETE FROM items");+ assert.equal((await client.query(itemsCopySelectedSql())).rows[0].rows, 1);+ assert.equal((await client.query(itemsSelectedRowsSql())).rows[0].text, "two");+ } finally {+ client.release();+ }+ });+});++test("items copy preserves missing call IDs as NULL and still constrains present IDs", { skip }, async () => {+ await withSchema(async (raw) => {+ await seed(raw, "nullable-calls", [+ ["tool_call", { tool: "execute", command: "echo hello" }],+ ["tool_result", { tool: "execute", result: "hello" }],+ ["tool_call", { tool: "execute", callId: "" }],+ ["tool_result", { tool: "execute", callId: null }],+ ["tool_call", { tool: "execute", callId: "real-call" }],+ ["tool_result", { tool: "execute", callId: "real-call" }],+ ]);+ await copyAll(raw, 1);+ const rows = await items(raw, "nullable-calls");+ assert.deepEqual(+ rows.map((row) => row.call_id),+ [null, null, null, null, "real-call", "real-call"],+ );+ assert.ok(rows.every((row) => !("callId" in row.body)));+ await assert.rejects(+ raw.query("UPDATE items SET call_id='' WHERE session_id='nullable-calls' AND seq=0"),+ /check constraint/,+ );+ await assert.rejects(+ raw.query("UPDATE items SET call_id='real-call' WHERE session_id='nullable-calls' AND seq=0"),+ /unique constraint/,+ );+ await assert.rejects(+ raw.query("UPDATE items SET kind='UserMessage',tool=NULL WHERE session_id='nullable-calls' AND seq=4"),+ /items_call_id_kind/,+ );+ });+});++test("payload parsing keeps ordinary JSON fast-path and Unicode repair semantics", { skip }, async () => {+ await withSchema(async (raw) => {+ const valid: Array<[string, unknown]> = [+ [+ String.raw`{"text":"ordinary \"quoted\" text","values":[1,true,null]}`,+ { text: 'ordinary "quoted" text', values: [1, true, null] },+ ],+ [String.raw`{"text":"\u0041"}`, { text: "A" }],+ [String.raw`{"text":"before\u0000after"}`, { text: "beforeafter" }],+ [String.raw`{"text":"before\ud800after"}`, { text: "before�after" }],+ [String.raw`{"text":"literal\\u0000"}`, { text: String.raw`literal\u0000` }],+ ];+ for (const [input, expected] of valid)+ assert.deepEqual(+ (await raw.query("SELECT items_copy_payload($1::text) AS parsed", [input])).rows[0].parsed,+ expected,+ );+ for (const input of [String.raw`{"text":"\q"}`, '{"text":"unfinished"', String.raw`{"text":"\u0000","bad":}`])+ await assert.rejects(raw.query("SELECT items_copy_payload($1::text)", [input]), /invalid input syntax/);+ });+});++test("items copy preserves stopped-turn records", { skip }, async () => {+ await withSchema(async (raw) => {+ const runId = "11111111-1111-4111-8111-111111111111";+ await seed(raw, "stopped", [["system", { kind: "turn_stopped", runId, reason: "user", text: "Stopped" }]]);+ await raw.query("INSERT INTO participants(session_id, principal_id, valid_from) VALUES ('stopped', 'u:ann', 0)");+ await raw.query(itemsCopySql());+ const [row] = await items(raw, "stopped");+ assert.ok(row);+ assert.equal(row.kind, "Stop");+ assert.equal(row.turn_id, runId);+ assert.equal(row.author, "u:ann");+ assert.deepEqual(row.body, { reason: "user", text: "Stopped" });+ });+});++test("items copy maps goal cancellation notices to system wakes", { skip }, async () => {+ await withSchema(async (raw) => {+ const body = { kind: "goal_cancelled", objective: "A sample goal", cancelledBy: "u:ann" };+ await seed(raw, "cancelled-goal", [["system", body]]);+ await raw.query(itemsCopySql());+ const [row] = await items(raw, "cancelled-goal");+ assert.ok(row);+ assert.equal(row.kind, "Wake");+ assert.equal(row.wake_cause, "system");+ assert.equal(row.author, null);+ assert.deepEqual(row.body, body);+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");+ await raw.query("INSERT INTO items_copy_batch VALUES ('cancelled-goal', 0)");+ const selected = (await raw.query(itemsSelectedRowsSql())).rows[0];+ assert.equal(selected.kind, "Wake");+ assert.equal(selected.wake_cause, "system");+ assert.deepEqual(selected.body, body);+ });+});++test("oversized legacy call identifiers become NULL without dropping history", { skip }, async () => {+ await withSchema(async (raw) => {+ const callId = "x".repeat(1025);+ const boundary = "x".repeat(1024);+ await seed(raw, "long-calls", [+ ["tool_call", { tool: "execute", callId, command: "echo preserved" }],+ ["tool_result", { tool: "execute", callId, result: "preserved output" }],+ ["tool_call", { tool: "execute", callId: boundary }],+ ["tool_result", { tool: "execute", callId: boundary, result: "boundary output" }],+ ["tool_call", { tool: "execute", callId: "é".repeat(513) }],+ ]);+ await raw.query(itemsCopySql());+ const rows = await items(raw, "long-calls");+ assert.deepEqual(+ rows.map((row) => row.call_id),+ [null, null, boundary, boundary, null],+ );+ assert.equal(rows[0]!.body.command, "echo preserved");+ assert.equal(rows[1]!.text, "preserved output");+ assert.equal(rows[3]!.text, "boundary output");+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");+ await raw.query("INSERT INTO items_copy_batch SELECT session_id,seq FROM session_entries");+ const selected = (await raw.query(itemsSelectedRowsSql())).rows;+ assert.deepEqual(+ selected.map((row) => row.call_id),+ [null, null, boundary, boundary, null],+ );+ });+});++test("ambiguous legacy IDs are cleared across calls and results, including across batches", { skip }, async () => {+ await withSchema(async (raw) => {+ await seed(raw, "ambiguous", [+ ["tool_call", { tool: "execute", callId: "repeated-result", command: "preserved" }],+ ["tool_result", { tool: "execute", callId: "repeated-result", result: "first" }],+ ["tool_result", { tool: "execute", callId: "repeated-result", result: "second" }],+ ["tool_call", { tool: "execute", callId: "repeated-call" }],+ ["tool_call", { tool: "execute", callId: "repeated-call" }],+ ["tool_result", { tool: "execute", callId: "repeated-call", result: "third" }],+ ["tool_call", { tool: "execute", callId: "valid-pair" }],+ ["tool_result", { tool: "execute", callId: "valid-pair", result: "valid" }],+ ]);+ await seed(raw, "other-session", [["tool_call", { tool: "execute", callId: "repeated-call" }]]);+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");+ await raw.query("INSERT INTO items_copy_batch VALUES ('ambiguous',0),('ambiguous',1)");+ await raw.query(itemsCopySelectedSql());+ assert.deepEqual(+ (await items(raw, "ambiguous")).map((row) => row.call_id),+ [null, null],+ );+ await raw.query("TRUNCATE items");+ await raw.query(itemsCopySql());+ const rows = await items(raw, "ambiguous");+ assert.deepEqual(+ rows.map((row) => row.call_id),+ [null, null, null, null, null, null, "valid-pair", "valid-pair"],+ );+ assert.equal(rows[0]!.body.command, "preserved");+ assert.deepEqual([rows[1]!.text, rows[2]!.text, rows[5]!.text], ["first", "second", "third"]);+ assert.equal((await items(raw, "other-session"))[0]!.call_id, "repeated-call");+ });+});diff --git a/test/postgres-items-operator.test.ts b/test/postgres-items-operator.test.tsnew file mode 100644index 000000000..8282e9a69--- /dev/null+++ b/test/postgres-items-operator.test.ts@@ -0,0 +1,203 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import pg from "pg";+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";++const url = process.env.DATABASE_URL;+const insertSession = `INSERT INTO sessions(id,type,scope_id,thread_ref,created_at) VALUES($1,'dm','personal:test',$1,1)`;+const insertEntry = `INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES($1,$2,'user',$3,'personal:test',1)`;++test(+ "items operator resumes bounded copies and reconciles updates, low keys, delayed commits and cascade deletes",+ { skip: !url, timeout: 45000 },+ async () => {+ const pool = new pg.Pool({ connectionString: url });+ const client = new pg.Client({ connectionString: url });+ const writer = new pg.Client({ connectionString: url });+ await client.connect();+ await writer.connect();+ try {+ createPostgresSessionStore(url!);+ createPostgresDirectoryStore(url!);+ await applyPgMigrations(+ pool,+ registeredPgMigrations(url!).filter((m) => m.id !== "sessions/store/0035-items"),+ );+ await client.query(+ "ALTER TABLE session_entries ADD CONSTRAINT operator_source_fk FOREIGN KEY(session_id) REFERENCES sessions(id) ON DELETE CASCADE",+ );+ await client.query(insertSession, ["z"]);+ for (let i = 0; i < 8; i++) await client.query(insertEntry, ["z", i, JSON.stringify({ text: `line ${i}` })]);+ await client.query("UPDATE session_entries SET type='soul' WHERE seq>=6");+ const op = new ItemsCopyOperator(client, 2);+ await op.prepare();+ await assert.rejects(op.prepare(), /already exists/);+ assert.equal(await op.copyBatch(), 2);+ await writer.query("UPDATE session_entries SET payload='broken' WHERE session_id='z' AND seq=2");+ const checkpoint = await op.status();+ await assert.rejects(op.copyBatch(), /invalid input syntax/);+ assert.equal((await op.status()).scanned, checkpoint.scanned);+ await writer.query("UPDATE session_entries SET payload='{\"text\":\"fixed\"}' WHERE session_id='z' AND seq=2");+ await writer.query(insertSession, ["a"]);+ await writer.query(insertEntry, ["a", 0, '{"text":"lower key"}']);+ await writer.query("UPDATE session_entries SET payload='{\"text\":\"edit\"}' WHERE session_id='z' AND seq=0");+ await writer.query("DELETE FROM session_entries WHERE session_id='z' AND seq=1");+ await writer.query("UPDATE session_entries SET seq=12 WHERE session_id='z' AND seq=4");+ await assert.rejects(writer.query("TRUNCATE session_entries"), /TRUNCATE is disabled/);+ const resumed = new ItemsCopyOperator(client, 2);+ while (await resumed.copyBatch()) continue;+ assert.equal(await resumed.completeCopy(), true);+ assert.equal((await resumed.status()).phase, "catchup");+ await writer.query("BEGIN");+ await writer.query(insertSession, ["late"]);+ await writer.query(insertEntry, ["late", 0, '{"text":"late commit"}']);+ await client.query("UPDATE session_entries SET payload='{\"text\":\"later id\"}' WHERE session_id='z' AND seq=5");+ while (await resumed.catchupBatch()) continue;+ await writer.query("COMMIT");+ while (await resumed.catchupBatch()) continue;+ assert.equal((await resumed.status()).queued, "0");+ assert.deepEqual(+ (await client.query("SELECT session_id,seq::int,text FROM items ORDER BY session_id,seq")).rows,+ [+ { session_id: "a", seq: 0, text: "lower key" },+ { session_id: "late", seq: 0, text: "late commit" },+ { session_id: "z", seq: 0, text: "edit" },+ { session_id: "z", seq: 2, text: "fixed" },+ { session_id: "z", seq: 3, text: "line 3" },+ { session_id: "z", seq: 5, text: "later id" },+ { session_id: "z", seq: 12, text: "line 4" },+ ],+ );+ await resumed.verify();+ await client.query("UPDATE items SET text='corrupt' WHERE session_id='z' AND seq=0");+ await assert.rejects(resumed.verify(), /verification found differences/);+ await assert.rejects(resumed.finish(true), /phase changed/);+ await writer.query("UPDATE session_entries SET payload=payload WHERE session_id='z' AND seq=0");+ while (await resumed.catchupBatch()) continue;+ await resumed.verify();+ await writer.query("DELETE FROM sessions WHERE id='a'");+ while (await resumed.catchupBatch()) continue;+ assert.equal((await client.query("SELECT count(*)::int AS n FROM items WHERE session_id='a'")).rows[0].n, 0);+ await resumed.verify();+ await client.query("INSERT INTO participants(session_id,principal_id,valid_from) VALUES('z','ann',0)");+ while (await resumed.catchupBatch()) continue;+ assert.equal(+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,+ "ann",+ );+ await client.query("INSERT INTO participants(session_id,principal_id,valid_from) VALUES('z','bob',0)");+ await client.query(+ 'UPDATE session_entries SET payload=\'{"text":"edit","name":"Ann","ts":"1.1"}\' WHERE session_id=\'z\' AND seq=0',+ );+ await client.query(+ "INSERT INTO directory_members(org_id,principal_id,display_name,display_name_lc,type) VALUES('o','ann','Ann','ann','internal')",+ );+ while (await resumed.catchupBatch()) continue;+ assert.equal(+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,+ "ann",+ );+ await client.query("DELETE FROM directory_members WHERE principal_id='ann'");+ while (await resumed.catchupBatch()) continue;+ assert.equal(+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,+ null,+ );+ await client.query(insertEntry, ["z", 15, '{"thinking":"thought"}']);+ await client.query("UPDATE session_entries SET type='thinking' WHERE session_id='z' AND seq=15");+ await client.query(+ 'INSERT INTO legacy_tape_steps(session_id,seq,step) VALUES(\'z\',15,\'{"index":3,"model":"m"}\')',+ );+ while (await resumed.catchupBatch()) continue;+ assert.equal((await client.query("SELECT step FROM items WHERE session_id='z' AND seq=15")).rows[0].step, 3);+ await client.query(insertEntry, ["z", 20, '{"kind":"message_revision","ts":"9.1"}']);+ await client.query("UPDATE session_entries SET type='system' WHERE session_id='z' AND seq=20");+ while (await resumed.catchupBatch()) continue;+ assert.equal(+ (await client.query("SELECT target_seq FROM items WHERE session_id='z' AND seq=20")).rows[0].target_seq,+ null,+ );+ await client.query(+ 'UPDATE session_entries SET payload=\'{"text":"new first","ts":"9.1"}\' WHERE session_id=\'z\' AND seq=0',+ );+ await resumed.catchupBatch();+ await client.query(+ 'UPDATE session_entries SET payload=\'{"text":"newer first","ts":"9.1"}\' WHERE session_id=\'z\' AND seq=0',+ );+ while (await resumed.catchupBatch()) continue;+ assert.equal(+ (await client.query("SELECT target_seq::int FROM items WHERE session_id='z' AND seq=20")).rows[0].target_seq,+ 0,+ );+ assert.equal(+ (await client.query("SELECT text FROM items WHERE session_id='z' AND seq=0")).rows[0].text,+ "newer first",+ );+ await resumed.verify();+ await client.query("UPDATE session_entries SET payload=payload WHERE session_id='late'");+ await writer.query("BEGIN");+ await writer.query("DELETE FROM sessions WHERE id='late'");+ await assert.rejects(resumed.catchupBatch(), (error) => (error as { code?: string }).code === "55P03");+ await writer.query("COMMIT");+ while (await resumed.catchupBatch()) continue;+ await resumed.verify();+ await assert.rejects(resumed.cleanup(), /cutover migration/);+ await assert.rejects(resumed.finish(false), /writers-stopped/);+ await client.query(+ "INSERT INTO session_leases(session_id,token,expires_at) VALUES('z','busy',9223372036854775807)",+ );+ await assert.rejects(resumed.finish(true), /active session leases/);+ await client.query("DELETE FROM session_leases WHERE session_id='z'");+ await client.query("ALTER TABLE session_entries DISABLE TRIGGER items_copy_capture");+ await assert.rejects(resumed.catchupBatch(), /capture is missing or disabled/);+ await client.query("ALTER TABLE session_entries ENABLE TRIGGER items_copy_capture");+ await resumed.verify();+ await assert.rejects(resumed.finish(true), /build the items indexes/);+ await assert.rejects(+ client.query("CREATE UNIQUE INDEX CONCURRENTLY items_one_call_per_kind ON items(tenant_id)"),+ /could not create unique index/,+ );+ await writer.query("SELECT pg_advisory_lock(hashtext('qm:schema-migrations'))");+ try {+ await resumed.indexes();+ await resumed.indexes();+ } finally {+ await writer.query("SELECT pg_advisory_unlock(hashtext('qm:schema-migrations'))");+ }+ assert.match(+ (await client.query("SELECT pg_get_indexdef('items_one_call_per_kind'::regclass) AS def")).rows[0].def,+ /call_id\) WHERE \(call_id IS NOT NULL\)/,+ );+ assert.equal(+ (+ await client.query(+ "SELECT count(*)::int AS n FROM pg_index WHERE indrelid='items'::regclass AND NOT indisvalid",+ )+ ).rows[0].n,+ 0,+ );+ await resumed.finish(true);+ assert.equal((await resumed.status()).phase, "ready");+ assert.equal(+ (await client.query("SELECT count(*)::int AS n FROM pg_indexes WHERE tablename='items'")).rows[0].n,+ 7,+ );+ await resumed.finish(true);+ await applyPgMigrations(pool, registeredPgMigrations(url!));+ await client.query(+ "INSERT INTO qm_schema_migrations(id,checksum) VALUES('sessions/store/0036-items-cutover','test')",+ );+ await assert.rejects(resumed.catchupBatch(), /cutover is already applied/);+ await resumed.cleanup();+ assert.equal((await client.query("SELECT to_regnamespace('items_copy_ops') AS schema")).rows[0].schema, null);+ assert.ok((await client.query("SELECT count(*)::int AS n FROM items")).rows[0].n > 0);+ } finally {+ await writer.end();+ await client.end();+ await pool.end();+ }+ },+);diff --git a/test/postgres-items-parallel.test.ts b/test/postgres-items-parallel.test.tsnew file mode 100644index 000000000..ad37de648--- /dev/null+++ b/test/postgres-items-parallel.test.ts@@ -0,0 +1,71 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import pg from "pg";+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";++const url = process.env.DATABASE_URL;++test(+ "sliced copy workers resume one cursor, copy every row once and wait out schema migrations",+ { skip: !url, timeout: 60000 },+ async () => {+ const pool = new pg.Pool({ connectionString: url });+ const clients = await Promise.all(+ Array.from({ length: 6 }, async () => {+ const c = new pg.Client({ connectionString: url });+ await c.connect();+ return c;+ }),+ );+ const [main, migrator, ...workers] = clients as [pg.Client, pg.Client, ...pg.Client[]];+ try {+ createPostgresSessionStore(url!);+ createPostgresDirectoryStore(url!);+ await applyPgMigrations(+ pool,+ registeredPgMigrations(url!).filter((m) => m.id !== "sessions/store/0035-items"),+ );+ await main.query(`INSERT INTO sessions(id,type,scope_id,thread_ref,created_at)+ SELECT 's'||lpad(i::text,3,'0'),'dm','personal:test','s'||i,1 FROM generate_series(0,199) i`);+ await main.query(`INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at)+ SELECT 's'||lpad((i%200)::text,3,'0'),i/200,'user',json_build_object('text','line '||i)::text,'personal:test',1+ FROM generate_series(0,1999) i`);+ const op = new ItemsCopyOperator(main, 50);+ await op.prepare();+ assert.equal(await op.copyBatch(), 50);+ assert.equal(await op.split(4), 4);+ assert.equal(await op.split(4), 4);+ await assert.rejects(op.split(3), /already split into 4 slices/);+ const slices = (await main.query("SELECT slice FROM items_copy_ops.slices ORDER BY slice")).rows.map(+ (r) => r.slice,+ );+ await migrator.query("SELECT pg_advisory_lock(hashtext('qm:schema-migrations'))");+ const release = setTimeout(+ () => void migrator.query("SELECT pg_advisory_unlock(hashtext('qm:schema-migrations'))"),+ 1500,+ );+ await Promise.all(+ slices.map(async (slice, i) => {+ const w = new ItemsCopyOperator(workers[i]!, 50);+ while (await w.copyBatch(slice)) continue;+ }),+ );+ clearTimeout(release);+ assert.equal(await op.completeCopy(), true);+ const status = await op.status();+ assert.deepEqual(+ [status.phase, Number(status.scanned), Number(status.initial_copied), status.done],+ ["catchup", 2000, 2000, 4],+ );+ assert.equal((await main.query("SELECT count(DISTINCT (session_id,seq))::int AS n FROM items")).rows[0].n, 2000);+ await op.verify();+ assert.equal((await op.status()).phase, "verified");+ } finally {+ await Promise.all(clients.map((c) => c.end()));+ await pool.end();+ }+ },+);