Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

History copy: items schema and resumable production backfill

Regan Bell2c6024dd37a8
diff --git a/package.json b/package.json
index 129d4fa85..7aaba0d2f 100644
--- a/package.json
+++ b/package.json
@@ -48,7 +48,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
+ "test:pg": "node scripts/run-pg-tests.mjs test/batch-paths.test.ts test/session-append-many.test.ts test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-tapeless-store.test.ts test/postgres-legacy-tape-steps.test.ts test/postgres-items-copy.test.ts test/semantic-memory-pg.test.ts test/semantic-memory-http.test.ts test/semantic-memory-large-pg.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-acl-file-id-migration.test.ts test/postgres-admin-grants.test.ts test/postgres-teams.test.ts test/postgres-decisions.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-linked-identity-membership.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-cron-run-as-migration.test.ts test/postgres-trigger-recipient-consent-migration.test.ts test/postgres-legacy-harness-migration.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-app-page-views.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/run-priority.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/plugins/web-ui/test/timeline.test.ts b/plugins/web-ui/test/timeline.test.ts
index a9c7e92ab..8b5465e22 100644
--- a/plugins/web-ui/test/timeline.test.ts
+++ b/plugins/web-ui/test/timeline.test.ts
@@ -586,3 +586,22 @@ test("live reasoning renders as a trailing thinking row until its persisted entr
assert.equal(buildTimeline(otherBlock).length, 3, "a different block of the same step does not hide this one");
assert.equal(buildTimeline({ ...work, status: "complete" }).length, 1, "finished work shows only durable rows");
});
+
+test("NULL legacy call IDs preserve the existing adjacent-result fallback", () => {
+ for (const callId of [null, "known-call"]) {
+ const work: WorkBlock = {
+ status: "complete",
+ activity: [
+ act(10, "tool_call", { tool: "execute", command: "echo hello", callId }),
+ act(11, "tool_result", { tool: "execute", result: "hello", code: 0, callId: null }),
+ ],
+ };
+ const timeline = buildTimeline(work);
+ assert.equal(timeline.length, 1);
+ assert.equal(timeline[0].kind, "tool");
+ if (timeline[0].kind !== "tool") throw new Error("expected paired tool row");
+ assert.equal(timeline[0].row.call?.seq, 10);
+ assert.equal(timeline[0].row.result?.seq, 11);
+ assert.equal(toolRowKind(timeline[0].row, work.status), "ok");
+ }
+});
diff --git a/scripts/ops/items-copy.ts b/scripts/ops/items-copy.ts
new file mode 100644
index 000000000..8ab4033a0
--- /dev/null
+++ b/scripts/ops/items-copy.ts
@@ -0,0 +1,529 @@
+import pg from "pg";
+import { pgMigrationChecksum } from "../../src/persistence/pg-pool.ts";
+import { createHash } from "node:crypto";
+import { parseArgs } from "node:util";
+import { pathToFileURL } from "node:url";
+import { sleep } from "../../src/util/async.ts";
+import { ITEMS_TABLES_SQL, ITEM_INDEXES, itemIndexSql, itemsMigration } from "../../src/sessions/items-schema.ts";
+import { itemsCopySelectedSql, itemsSelectedRowsSql, ITEMS_COPY_REPAIR_SQL } from "../../src/sessions/items-copy.ts";
+
+const fingerprint = createHash("sha256")
+ .update(JSON.stringify([ITEMS_TABLES_SQL, ITEM_INDEXES, ITEMS_COPY_REPAIR_SQL, itemsCopySelectedSql()]))
+ .digest("hex");
+const state = "items_copy_ops.state";
+const dirty = "items_copy_ops.dirty";
+const slices = "items_copy_ops.slices";
+const SLICES_SQL = `CREATE TABLE IF NOT EXISTS ${slices}(slice int PRIMARY KEY,hi_session text,cursor_session text,cursor_seq int,
+ scanned bigint NOT NULL DEFAULT 0,copied bigint NOT NULL DEFAULT 0,done boolean NOT NULL DEFAULT false)`;
+
+export class ItemsCopyOperator {
+ readonly client: pg.Client;
+ readonly batch: number;
+ constructor(client: pg.Client, batch = 2000) {
+ this.client = client;
+ this.batch = batch;
+ if (!Number.isSafeInteger(batch) || batch < 1 || batch > 10000)
+ throw new Error("batch must be between 1 and 10000");
+ }
+
+ async transaction<T>(
+ run: () => Promise<T>,
+ migrationLock: "exclusive" | "shared" | "none" = "exclusive",
+ ): Promise<T> {
+ await this.client.query(
+ "BEGIN; SET LOCAL lock_timeout='2s'; SET LOCAL statement_timeout='120s'; SET LOCAL search_path=public",
+ );
+ try {
+ const lock = `SELECT pg_try_advisory_xact_lock${migrationLock === "shared" ? "_shared" : ""}(hashtext('qm:schema-migrations')) AS acquired`;
+ for (let attempt = 0; migrationLock !== "none" && !(await this.client.query(lock)).rows[0].acquired; attempt++) {
+ if (attempt >= 300) throw new Error("schema migration is running; retry after it completes");
+ await sleep(1000);
+ }
+ const result = await run();
+ await this.client.query("COMMIT");
+ return result;
+ } catch (error) {
+ try {
+ await this.client.query("ROLLBACK");
+ } catch (rollback) {
+ throw new AggregateError([error, rollback], "items copy failed and rollback failed", { cause: rollback });
+ }
+ throw error;
+ }
+ }
+
+ async prepare() {
+ await this.transaction(async () => {
+ if ((await this.client.query("SELECT to_regclass('public.items') AS existing")).rows[0].existing)
+ throw new Error("items already exists; inspect it before preparing a new copy");
+ await this.client.query(
+ "LOCK TABLE public.session_entries, public.participants, public.directory_members, public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",
+ );
+ await this.client.query("CREATE SCHEMA items_copy_ops");
+ for (const sql of ITEMS_TABLES_SQL) await this.client.query(sql);
+ for (const sql of ITEMS_COPY_REPAIR_SQL) await this.client.query(sql);
+ await this.client.query("CREATE EXTENSION IF NOT EXISTS btree_gin");
+ await this.client.query(`CREATE TABLE ${state}(singleton boolean PRIMARY KEY DEFAULT true CHECK(singleton),
+ source_oid oid NOT NULL, target_oid oid NOT NULL, tenant_id uuid NOT NULL, fingerprint text NOT NULL,
+ phase text NOT NULL, cursor_session text, cursor_seq int, scanned bigint NOT NULL DEFAULT 0,
+ copied bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now())`);
+ await this.client.query(
+ `INSERT INTO ${state}(source_oid,target_oid,tenant_id,fingerprint,phase)
+ VALUES('public.session_entries'::regclass,'public.items'::regclass,(SELECT id FROM public.qm_tenant),$1,'copy')`,
+ [fingerprint],
+ );
+ await this.client.query(
+ `CREATE TABLE ${dirty}(session_id text PRIMARY KEY,version bigint NOT NULL DEFAULT 1,copy_version bigint,cursor_seq int,touched_at timestamptz NOT NULL DEFAULT now())`,
+ );
+ await this.client.query(`CREATE INDEX dirty_oldest ON ${dirty}(touched_at,session_id)`);
+ await this.client.query(SLICES_SQL);
+ await this.client.query(`INSERT INTO ${slices}(slice) VALUES(0)`);
+ await this.client.query(`CREATE FUNCTION items_copy_ops.capture() RETURNS trigger LANGUAGE plpgsql AS $capture$
+ BEGIN
+ IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'TRUNCATE is disabled during items copy'; END IF;
+ IF TG_TABLE_NAME='directory_members' THEN
+ IF TG_OP='UPDATE' AND (OLD.principal_id,OLD.display_name_lc) IS NOT DISTINCT FROM (NEW.principal_id,NEW.display_name_lc) THEN RETURN NULL; END IF;
+ INSERT INTO ${dirty}(session_id)
+ SELECT DISTINCT session_id FROM participants
+ WHERE principal_id IN (CASE WHEN TG_OP<>'INSERT' THEN OLD.principal_id END,CASE WHEN TG_OP<>'DELETE' THEN NEW.principal_id END)
+ ORDER BY session_id
+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;
+ ELSE
+ IF TG_TABLE_NAME='participants' AND TG_OP='UPDATE' THEN
+ IF (OLD.principal_id,OLD.session_id,OLD.valid_from,OLD.valid_to) IS NOT DISTINCT FROM (NEW.principal_id,NEW.session_id,NEW.valid_from,NEW.valid_to) THEN RETURN NULL; END IF;
+ END IF;
+ IF TG_OP<>'INSERT' THEN
+ INSERT INTO ${dirty}(session_id) VALUES(OLD.session_id)
+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;
+ END IF;
+ IF TG_OP<>'DELETE' THEN
+ INSERT INTO ${dirty}(session_id) VALUES(NEW.session_id)
+ ON CONFLICT(session_id) DO UPDATE SET version=${dirty}.version+1;
+ END IF;
+ END IF;
+ RETURN NULL;
+ END $capture$`);
+ for (const table of ["session_entries", "participants", "directory_members", "legacy_tape_steps"]) {
+ await this.client.query(
+ `CREATE TRIGGER items_copy_capture AFTER INSERT OR UPDATE OR DELETE ON public.${table} FOR EACH ROW EXECUTE FUNCTION items_copy_ops.capture()`,
+ );
+ await this.client.query(
+ `CREATE TRIGGER items_copy_no_truncate BEFORE TRUNCATE ON public.${table} FOR EACH STATEMENT EXECUTE FUNCTION items_copy_ops.capture()`,
+ );
+ }
+ });
+ }
+
+ async assertState(phases: string[], lock: "UPDATE" | "SHARE" = "UPDATE") {
+ const {
+ rows: [cutover],
+ } = await this.client.query(
+ "SELECT EXISTS(SELECT 1 FROM public.qm_schema_migrations WHERE id LIKE 'sessions/store/%items-cutover') AS done",
+ );
+ if (cutover.done) throw new Error("items cutover is already applied; the legacy copy operator must not run");
+ const {
+ rows: [s],
+ } = await this.client.query(
+ `SELECT *,source_oid='public.session_entries'::regclass AND target_oid='public.items'::regclass AND tenant_id=(SELECT id FROM public.qm_tenant) AS same FROM ${state} FOR ${lock}`,
+ );
+ if (!s?.same || s.fingerprint !== fingerprint || !phases.includes(s.phase))
+ throw new Error("copy source, target, mapping or phase changed; inspect status");
+ const {
+ rows: [t],
+ } = await this.client.query(
+ "SELECT count(*)::int AS n FROM pg_trigger WHERE tgrelid IN ('public.session_entries'::regclass,'public.participants'::regclass,'public.directory_members'::regclass,'public.legacy_tape_steps'::regclass) AND tgname IN ('items_copy_capture','items_copy_no_truncate') AND tgenabled='O'",
+ );
+ if (t.n !== 8) throw new Error("items copy change capture is missing or disabled");
+ await this.client.query(
+ "CREATE TEMP TABLE IF NOT EXISTS items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq)) ON COMMIT DELETE ROWS",
+ );
+ return s;
+ }
+
+ async replaceSelected() {
+ await this.client.query(
+ "SELECT id FROM public.sessions WHERE id IN (SELECT session_id FROM pg_temp.items_copy_batch) ORDER BY id FOR KEY SHARE",
+ );
+ await this.client.query(
+ "DELETE FROM public.items i USING pg_temp.items_copy_batch b WHERE i.tenant_id=(SELECT id FROM public.qm_tenant) AND (i.session_id,i.seq)=(b.session_id,b.seq)",
+ );
+ return (await this.client.query(itemsCopySelectedSql())).rows[0];
+ }
+
+ async split(workers: number) {
+ if (!Number.isSafeInteger(workers) || workers < 1 || workers > 32)
+ throw new Error("workers must be between 1 and 32");
+ return this.transaction(async () => {
+ const s = await this.assertState(["copy"]);
+ if (!(await this.client.query(`SELECT to_regclass('${slices}') AS t`)).rows[0].t) {
+ await this.client.query(SLICES_SQL);
+ await this.client.query(`INSERT INTO ${slices}(slice,cursor_session,cursor_seq) VALUES(0,$1,$2)`, [
+ s.cursor_session,
+ s.cursor_seq,
+ ]);
+ }
+ const { rows } = await this.client.query(`SELECT * FROM ${slices} ORDER BY slice FOR UPDATE`);
+ const open = rows.filter((r) => !r.done);
+ if (rows.length === workers || workers === 1) return rows.length;
+ if (open.length !== 1)
+ throw new Error(`copy is already split into ${rows.length} slices; rerun with --workers ${rows.length}`);
+ const [from] = open;
+ const {
+ rows: [{ reltuples }],
+ } = await this.client.query("SELECT reltuples FROM pg_class WHERE oid='public.session_entries'::regclass");
+ const {
+ rows: [{ bounds }],
+ } = await this.client.query(
+ `SELECT COALESCE(array_agg(DISTINCT b ORDER BY b),'{}') AS bounds FROM unnest((
+ SELECT percentile_disc((SELECT array_agg(i::float8/$1) FROM generate_series(1,$1-1) i)) WITHIN GROUP (ORDER BY session_id)
+ FROM public.session_entries TABLESAMPLE SYSTEM($2)
+ WHERE ($3::text IS NULL OR session_id>$3) AND ($4::text IS NULL OR session_id<$4))) b
+ WHERE b IS NOT NULL AND ($3::text IS NULL OR b>$3)`,
+ [workers, Number(reltuples) > 1e6 ? 0.2 : 100, from.cursor_session, from.hi_session],
+ );
+ const his = [...bounds, from.hi_session];
+ const base = Math.max(...rows.map((r) => r.slice)) + 1;
+ await this.client.query(`UPDATE ${slices} SET hi_session=$2 WHERE slice=$1`, [from.slice, his[0]]);
+ for (const [i, lo] of bounds.entries())
+ await this.client.query(
+ `INSERT INTO ${slices}(slice,hi_session,cursor_session,cursor_seq) VALUES($1,$2,$3,-1)`,
+ [base + i, his[i + 1], lo],
+ );
+ return rows.length + bounds.length;
+ });
+ }
+
+ async copyBatch(slice = 0) {
+ return this.transaction(async () => {
+ await this.assertState(["copy"], "SHARE");
+ const {
+ rows: [sl],
+ } = await this.client.query(`SELECT * FROM ${slices} WHERE slice=$1 FOR UPDATE`, [slice]);
+ if (!sl || sl.done) return 0;
+ const params: unknown[] = [];
+ const where: string[] = [];
+ if (sl.cursor_session !== null)
+ where.push(`(session_id,seq)>($${params.push(sl.cursor_session)},$${params.push(sl.cursor_seq)})`);
+ if (sl.hi_session !== null) where.push(`session_id<$${params.push(sl.hi_session)}`);
+ await this.client.query(
+ `INSERT INTO pg_temp.items_copy_batch SELECT session_id,seq FROM public.session_entries
+ ${where.length ? `WHERE ${where.join(" AND ")}` : ""} ORDER BY session_id,seq LIMIT $${params.push(this.batch)}`,
+ params,
+ );
+ const {
+ rows: [last],
+ } = await this.client.query(
+ "SELECT session_id,seq,(SELECT count(*)::int FROM pg_temp.items_copy_batch) AS n FROM pg_temp.items_copy_batch ORDER BY session_id DESC,seq DESC LIMIT 1",
+ );
+ if (!last) {
+ await this.client.query(`UPDATE ${slices} SET done=true WHERE slice=$1`, [slice]);
+ return 0;
+ }
+ const copied = await this.replaceSelected();
+ await this.client.query(
+ `UPDATE ${slices} SET cursor_session=$2,cursor_seq=$3,scanned=scanned+$4,copied=copied+$5 WHERE slice=$1`,
+ [slice, last.session_id, last.seq, last.n, copied.rows],
+ );
+ return last.n as number;
+ }, "shared");
+ }
+
+ async completeCopy() {
+ return this.transaction(async () => {
+ await this.assertState(["copy"]);
+ const {
+ rows: [open],
+ } = await this.client.query(`SELECT count(*)::int AS n FROM ${slices} WHERE NOT done`);
+ if (open.n) return false;
+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);
+ return true;
+ });
+ }
+
+ async catchupBatch() {
+ return this.transaction(async () => {
+ await this.assertState(["catchup", "verified", "ready"]);
+ return this.drainBatch();
+ });
+ }
+
+ async drainBatch() {
+ await this.client.query("TRUNCATE pg_temp.items_copy_batch");
+ const {
+ rows: [entry],
+ } = await this.client.query(`SELECT * FROM ${dirty} ORDER BY touched_at,session_id LIMIT 1`);
+ if (!entry) return 0;
+ const version = entry.copy_version ?? entry.version;
+ const params = [entry.session_id, entry.cursor_seq ?? -1, this.batch];
+ await this.client.query(
+ `INSERT INTO pg_temp.items_copy_batch
+ WITH a AS (SELECT seq FROM public.session_entries WHERE session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3),
+ b AS (SELECT seq::int FROM public.items WHERE tenant_id=(SELECT id FROM public.qm_tenant) AND session_id=$1 AND seq>$2 ORDER BY seq LIMIT $3)
+ SELECT $1,seq FROM (SELECT seq FROM a UNION SELECT seq FROM b) keys ORDER BY seq LIMIT $3`,
+ params,
+ );
+ const {
+ rows: [batch],
+ } = await this.client.query("SELECT max(seq) AS last,count(*)::int AS n FROM pg_temp.items_copy_batch");
+ if (batch.n) {
+ await this.replaceSelected();
+ await this.client.query(
+ `UPDATE ${dirty} SET copy_version=$2,cursor_seq=$3,touched_at=clock_timestamp() WHERE session_id=$1`,
+ [entry.session_id, version, batch.last],
+ );
+ } else {
+ await this.client.query(`DELETE FROM ${dirty} WHERE session_id=$1 AND version=$2`, [entry.session_id, version]);
+ await this.client.query(
+ `UPDATE ${dirty} SET copy_version=NULL,cursor_seq=NULL,touched_at=clock_timestamp() WHERE session_id=$1`,
+ [entry.session_id],
+ );
+ }
+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);
+ return batch.n || 1;
+ }
+
+ async indexes() {
+ await this.transaction(async () => {
+ await this.assertState(["catchup", "verified"]);
+ }, "none");
+ await this.client.query(
+ "SET search_path=public; SET statement_timeout=0; SET lock_timeout='5s'; SET maintenance_work_mem='1GB'",
+ );
+ try {
+ for (const index of ITEM_INDEXES) {
+ const {
+ rows: [existing],
+ } = await this.client.query(
+ "SELECT i.indisvalid, i.indrelid='public.items'::regclass AS ours FROM pg_index i WHERE indexrelid=to_regclass($1)",
+ [`public.${index.name}`],
+ );
+ if (existing && !existing.ours) throw new Error(`index ${index.name} belongs to another table`);
+ if (existing?.indisvalid) continue;
+ if (existing) await this.client.query(`DROP INDEX public.${index.name}`);
+ await this.client.query(itemIndexSql(index));
+ }
+ } finally {
+ await this.client.query("RESET maintenance_work_mem");
+ }
+ }
+
+ async finish(writersStopped: boolean) {
+ if (!writersStopped)
+ throw new Error("finish requires --writers-stopped; keep old writers stopped through the items cutover");
+ await this.transaction(async () => {
+ await this.client.query("SET LOCAL statement_timeout=0");
+ await this.client.query(
+ "LOCK TABLE public.session_entries,public.participants,public.directory_members,public.legacy_tape_steps IN SHARE ROW EXCLUSIVE MODE",
+ );
+ await this.assertState(["verified", "ready"]);
+ const {
+ rows: [leases],
+ } = await this.client.query(
+ "SELECT count(*)::int AS n FROM public.session_leases WHERE expires_at>(extract(epoch FROM clock_timestamp())*1000)::bigint",
+ );
+ if (leases.n) throw new Error("active session leases remain; drain writers before finishing the copy");
+ while (await this.drainBatch()) continue;
+ const {
+ rows: [{ valid }],
+ } = await this.client.query(
+ `SELECT count(*)::int AS valid FROM pg_index i JOIN pg_class c ON c.oid=i.indexrelid
+ WHERE i.indrelid='public.items'::regclass AND i.indisvalid AND c.relname=ANY($1)`,
+ [ITEM_INDEXES.map((index) => index.name)],
+ );
+ if (valid !== ITEM_INDEXES.length)
+ throw new Error("build the items indexes before stopping writers: run indexes");
+ const checksum = pgMigrationChecksum(itemsMigration.statements);
+ const {
+ rows: [recorded],
+ } = await this.client.query("SELECT checksum FROM public.qm_schema_migrations WHERE id=$1", [itemsMigration.id]);
+ if (recorded && recorded.checksum !== checksum)
+ throw new Error("items migration checksum differs from the staged copy");
+ await this.client.query(
+ "INSERT INTO public.qm_schema_migrations(id,checksum) VALUES($1,$2) ON CONFLICT(id) DO NOTHING",
+ [itemsMigration.id, checksum],
+ );
+ await this.client.query(`UPDATE ${state} SET phase='ready',updated_at=now()`);
+ });
+ }
+
+ async verify() {
+ await this.transaction(async () => {
+ await this.assertState(["catchup", "verified", "ready"]);
+ await this.client.query(`UPDATE ${state} SET phase='catchup',updated_at=now()`);
+ });
+ for (const table of ["public.session_entries", "public.items"]) {
+ let cursor: [string, number] | null = null;
+ for (;;) {
+ const last = await this.transaction(async () => {
+ await this.assertState(["catchup", "verified", "ready"]);
+ const params = cursor === null ? [this.batch] : [...cursor, this.batch];
+ const where = [
+ table === "public.items" ? "tenant_id=(SELECT id FROM public.qm_tenant)" : "",
+ cursor === null ? "" : "(session_id,seq)>($1,$2)",
+ ]
+ .filter(Boolean)
+ .join(" AND ");
+ await this.client.query(
+ `INSERT INTO pg_temp.items_copy_batch SELECT session_id,seq FROM ${table}
+ ${where ? `WHERE ${where}` : ""} ORDER BY session_id,seq LIMIT $${params.length}`,
+ params,
+ );
+ const {
+ rows: [row],
+ } = await this.client.query(
+ "SELECT session_id,seq FROM pg_temp.items_copy_batch ORDER BY session_id DESC,seq DESC LIMIT 1",
+ );
+ if (!row) return null;
+ const expected = itemsSelectedRowsSql();
+ const {
+ rows: [diff],
+ } = await this.client.query(`WITH expected AS (${expected}),
+ actual AS (SELECT i.* FROM public.items i JOIN pg_temp.items_copy_batch b USING(session_id,seq)
+ WHERE tenant_id=(SELECT id FROM public.qm_tenant))
+ SELECT count(*)::int AS n FROM (
+ SELECT COALESCE(e.session_id,a.session_id) AS session_id,COALESCE(e.seq,a.seq) AS seq
+ FROM expected e FULL JOIN actual a USING(tenant_id,session_id,seq)
+ WHERE (to_jsonb(e)-'search'-'blob_sha') IS DISTINCT FROM (to_jsonb(a)-'search'-'blob_sha')
+ ) d WHERE NOT EXISTS(SELECT 1 FROM ${dirty} q WHERE q.session_id=d.session_id)`);
+ if (diff.n) throw new Error("items verification found differences without captured changes");
+ return [row.session_id, row.seq] as [string, number];
+ });
+ if (!last) break;
+ cursor = last;
+ }
+ }
+ await this.transaction(async () => {
+ await this.assertState(["catchup", "verified", "ready"]);
+ const {
+ rows: [duplicates],
+ } = await this.client.query(`SELECT EXISTS(
+ SELECT 1 FROM public.items WHERE call_id IS NOT NULL GROUP BY tenant_id,session_id,kind,call_id HAVING count(*)>1
+ ) AS found`);
+ if (duplicates.found) throw new Error("duplicate call IDs prevent the final items unique index");
+ await this.client.query(`UPDATE ${state} SET phase='verified',updated_at=now()`);
+ });
+ }
+
+ async cleanup() {
+ await this.transaction(async () => {
+ const {
+ rows: [cutover],
+ } = await this.client.query(
+ "SELECT EXISTS(SELECT 1 FROM public.qm_schema_migrations WHERE id LIKE 'sessions/store/%items-cutover') AS done",
+ );
+ if (!cutover.done) throw new Error("cleanup requires the items cutover migration to be applied");
+ for (const table of [
+ "session_entries",
+ "legacy_session_entries",
+ "participants",
+ "directory_members",
+ "legacy_tape_steps",
+ ]) {
+ if ((await this.client.query("SELECT to_regclass($1) AS relation", [`public.${table}`])).rows[0].relation) {
+ await this.client.query(`DROP TRIGGER IF EXISTS items_copy_capture ON public.${table}`);
+ await this.client.query(`DROP TRIGGER IF EXISTS items_copy_no_truncate ON public.${table}`);
+ }
+ }
+ await this.client.query("DROP FUNCTION IF EXISTS items_copy_ops.capture()");
+ await this.client.query(`DROP TABLE IF EXISTS ${dirty},${state},${slices}`);
+ await this.client.query("DROP SCHEMA IF EXISTS items_copy_ops");
+ });
+ }
+
+ async report() {
+ const { rows } = await this.client.query(`SELECT kind,count(*)::bigint AS rows,
+ count(*) FILTER (WHERE author IS NULL)::bigint AS author_null,
+ CASE WHEN kind IN ('UserMessage','SteeringMessage','ElicitationResponse') THEN
+ round(100.0*count(*) FILTER (WHERE author IS NULL)/count(*),2) END AS author_null_percent
+ FROM public.items GROUP BY kind ORDER BY kind`);
+ return rows;
+ }
+
+ async status() {
+ return (
+ await this.client
+ .query(`SELECT st.phase,st.scanned+sl.scanned AS scanned,st.copied+sl.copied AS initial_copied,sl.slices,sl.done,
+ st.updated_at,(SELECT count(*) FROM ${dirty}) AS queued,pg_total_relation_size('public.items') AS bytes
+ FROM ${state} st,(SELECT COALESCE(sum(scanned),0) AS scanned,COALESCE(sum(copied),0) AS copied,count(*)::int AS slices,
+ count(*) FILTER (WHERE done)::int AS done FROM ${slices}) sl`)
+ ).rows[0];
+ }
+}
+
+async function main() {
+ const { values, positionals } = parseArgs({
+ allowPositionals: true,
+ options: {
+ batch: { type: "string", default: "2000" },
+ workers: { type: "string", default: "1" },
+ "writers-stopped": { type: "boolean", default: false },
+ },
+ });
+ const action = positionals[0];
+ if (
+ !["prepare", "copy", "catchup", "indexes", "verify", "status", "report", "finish", "cleanup"].includes(action ?? "")
+ )
+ throw new Error(
+ "usage: node scripts/ops/items-copy.ts prepare|copy|catchup|indexes|verify|status|report|finish|cleanup [--batch 2000] [--workers 1] [--writers-stopped]",
+ );
+ if (!process.env.DATABASE_URL) throw new Error("DATABASE_URL must name the direct Postgres endpoint");
+ const client = new pg.Client({ connectionString: process.env.DATABASE_URL, application_name: "qm-items-copy" });
+ await client.connect();
+ try {
+ const op = new ItemsCopyOperator(client, Number(values.batch));
+ if (action !== "status") {
+ const {
+ rows: [lock],
+ } = await client.query("SELECT pg_try_advisory_lock(hashtext('qm:items-copy')) AS acquired");
+ if (!lock.acquired) throw new Error("another items copy operator is running");
+ }
+ if (action === "cleanup") {
+ await op.cleanup();
+ console.log("items copy temporary state removed");
+ return;
+ }
+ if (action === "prepare") await op.prepare();
+ if (action === "indexes") await op.indexes();
+ if (action === "verify") await op.verify();
+ if (action === "finish") await op.finish(values["writers-stopped"]);
+ if (action === "report") console.log(JSON.stringify(await op.report()));
+ if (action === "copy") {
+ await op.split(Number(values.workers));
+ let failed: unknown = null;
+ await Promise.all(
+ (await client.query(`SELECT slice FROM ${slices} WHERE NOT done ORDER BY slice`)).rows.map(
+ async ({ slice }) => {
+ const worker = new pg.Client({
+ connectionString: process.env.DATABASE_URL,
+ application_name: "qm-items-copy",
+ });
+ await worker.connect();
+ try {
+ const w = new ItemsCopyOperator(worker, op.batch);
+ for (let batches = 1; failed === null && (await w.copyBatch(slice)); batches++)
+ if (batches % 20 === 0) console.log(JSON.stringify({ slice, ...(await w.status()) }));
+ } catch (error) {
+ failed ??= error;
+ } finally {
+ await worker.end();
+ }
+ },
+ ),
+ );
+ if (failed !== null) throw failed;
+ await op.completeCopy();
+ }
+ if (action === "catchup") {
+ let batches = 0;
+ while (await op.catchupBatch()) if (++batches % 20 === 0) console.log(JSON.stringify(await op.status()));
+ }
+ console.log(JSON.stringify(await op.status()));
+ } finally {
+ await client.end();
+ }
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
+ main().catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ });
diff --git a/src/directory/postgres-directory-store.ts b/src/directory/postgres-directory-store.ts
index 6b969a1aa..cf6e33f3f 100644
--- a/src/directory/postgres-directory-store.ts
+++ b/src/directory/postgres-directory-store.ts
@@ -170,25 +170,27 @@ function dedupMemberships(rows: ChannelMembership[]): ChannelMembership[] {
);
}
+export const DIRECTORY_MIGRATIONS = [
+ {
+ id: "directory/store/0001",
+ expectedChecksum: "c47a45848f9d225ff601c9df13dc272349c7df7525e7e2417f33d68be7672cc1",
+ statements: INITIAL_SCHEMA,
+ },
+ { id: "directory/store/0002", statements: SYNC_STAMP_SCHEMA },
+ { id: "directory/store/0003", statements: EXTERNAL_ROSTER_SCHEMA },
+ {
+ id: "directory/store/0004",
+ statements: ["ALTER TABLE directory_channels ADD COLUMN IF NOT EXISTS observed_at BIGINT"],
+ },
+ {
+ id: "directory/store/0005",
+ statements: ["ALTER TABLE directory_members ADD COLUMN IF NOT EXISTS avatar_url TEXT"],
+ },
+];
+
export function createPostgresDirectoryStore(connectionString: string): DirectoryStore {
const orgId = configOrgId();
- const { q, pool } = createPgPool(connectionString, [
- {
- id: "directory/store/0001",
- expectedChecksum: "c47a45848f9d225ff601c9df13dc272349c7df7525e7e2417f33d68be7672cc1",
- statements: INITIAL_SCHEMA,
- },
- { id: "directory/store/0002", statements: SYNC_STAMP_SCHEMA },
- { id: "directory/store/0003", statements: EXTERNAL_ROSTER_SCHEMA },
- {
- id: "directory/store/0004",
- statements: ["ALTER TABLE directory_channels ADD COLUMN IF NOT EXISTS observed_at BIGINT"],
- },
- {
- id: "directory/store/0005",
- statements: ["ALTER TABLE directory_members ADD COLUMN IF NOT EXISTS avatar_url TEXT"],
- },
- ]);
+ const { q, pool } = createPgPool(connectionString, DIRECTORY_MIGRATIONS);
async function pick<T>(
query: string,
diff --git a/src/sessions/items-copy.ts b/src/sessions/items-copy.ts
new file mode 100644
index 000000000..ab6e34d4c
--- /dev/null
+++ b/src/sessions/items-copy.ts
@@ -0,0 +1,237 @@
+export const ITEMS_COPY_REPAIR_SQL = [
+ String.raw`CREATE OR REPLACE FUNCTION items_copy_payload(t text) RETURNS jsonb
+ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE AS $items_copy_payload$
+ DECLARE r text;
+ BEGIN
+ IF t IS NULL OR strpos(t, E'\\u') = 0 THEN RETURN t::jsonb; END IF;
+ IF pg_input_is_valid(t, 'jsonb') THEN RETURN t::jsonb; END IF;
+ r := regexp_replace(t, '(?<!\\)((?:\\\\)*)\\u0000', '\1', 'g');
+ IF NOT pg_input_is_valid(r, 'jsonb') THEN
+ r := regexp_replace(regexp_replace(r,
+ '(?<!\\)((?:\\\\)*)\\u[dD][89abAB][0-9a-fA-F]{2}(?!\\u[dD][c-fC-F][0-9a-fA-F]{2})', '\1\\ufffd', 'g'),
+ '(?<!\\)((?:\\\\)*)(?<!\\u[dD][89abAB][0-9a-fA-F]{2})\\u[dD][c-fC-F][0-9a-fA-F]{2}', '\1\\ufffd', 'g');
+ END IF;
+ RETURN r::jsonb;
+ END $items_copy_payload$`,
+ `CREATE OR REPLACE FUNCTION items_copy_payload(j jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS 'SELECT j'`,
+];
+
+export interface ItemsCopyOptions {
+ delta?: boolean;
+ bucket?: { index: number; count: number };
+}
+
+const bucketPredicate = (col: string, bucket: { index: number; count: number }): string => {
+ const { index, count } = bucket;
+ if (!Number.isInteger(count) || count < 1 || !Number.isInteger(index) || index < 0 || index >= count)
+ throw new Error(`items copy: bucket ${index} of ${count} is invalid`);
+ return `((hashtextextended(${col}, 0) % ${count}) + ${count}) % ${count} = ${index}`;
+};
+
+const TENANT = `(SELECT id FROM qm_tenant)`;
+const AUTHOR_KINDS_SQL = `('UserMessage', 'SteeringMessage', 'ElicitationResponse', 'Stop')`;
+
+const str = (key: string): string =>
+ `CASE WHEN jsonb_typeof(n.body -> '${key}') = 'string' THEN n.body ->> '${key}' END`;
+const isType = (key: string, type: string): string => `jsonb_typeof(n.body -> '${key}') = '${type}'`;
+
+const NORMALIZE = ` CROSS JOIN LATERAL (SELECT items_copy_payload(e.payload) AS j OFFSET 0) parsed
+ CROSS JOIN LATERAL (
+ SELECT CASE WHEN parsed.j IS NULL OR jsonb_typeof(parsed.j) = 'null' THEN '{}'::jsonb
+ WHEN jsonb_typeof(parsed.j) = 'object' THEN parsed.j
+ WHEN jsonb_typeof(parsed.j) = 'string' THEN jsonb_build_object('text', parsed.j)
+ ELSE jsonb_build_object('value', parsed.j)
+ END AS body
+ OFFSET 0
+ ) n
+`;
+
+function ambiguousCallsSql(selected: boolean): string {
+ return `SELECT DISTINCT session_id, call_id FROM (
+ SELECT e.session_id, e.type, n.body ->> 'callId' AS call_id
+ FROM session_entries e
+ ${NORMALIZE}
+ WHERE e.type IN ('tool_call', 'tool_result')
+ ${selected ? "AND e.session_id IN (SELECT DISTINCT session_id FROM pg_temp.items_copy_batch)" : ""}
+ AND jsonb_typeof(n.body -> 'callId') = 'string'
+ AND octet_length(n.body ->> 'callId') BETWEEN 1 AND 1024
+ GROUP BY e.session_id, e.type, n.body ->> 'callId'
+ HAVING count(*) > 1
+ ) repeated`;
+}
+
+const CALL_IDS = ` LEFT JOIN ambiguous_calls ambiguous
+ ON ambiguous.session_id=e.session_id AND ambiguous.call_id=n.body ->> 'callId'
+`;
+
+const KIND = ` CROSS JOIN LATERAL (
+ SELECT (CASE e.type
+ WHEN 'user' THEN CASE WHEN n.body @> '{"steered":true}' THEN 'SteeringMessage' ELSE 'UserMessage' END
+ WHEN 'assistant' THEN 'AgentMessage'
+ WHEN 'text' THEN 'AgentMessage'
+ WHEN 'delivery' THEN 'AgentMessage'
+ WHEN 'thinking' THEN 'Reasoning'
+ WHEN 'tool_call' THEN 'ToolCall'
+ WHEN 'tool_result' THEN 'ToolResult'
+ WHEN 'approval_request' THEN 'Elicitation'
+ WHEN 'approval_resolved' THEN 'ElicitationResponse'
+ WHEN 'soul' THEN NULL
+ WHEN 'text_start' THEN NULL
+ WHEN 'system' THEN
+ CASE n.body ->> 'kind'
+ WHEN 'context_summary' THEN 'ContextCompaction'
+ WHEN 'turn_failure' THEN 'Error'
+ WHEN 'turn_stopped' THEN 'Stop'
+ WHEN 'wake' THEN 'Wake'
+ WHEN 'goal_cancelled' THEN 'Wake'
+ WHEN 'message_revision' THEN 'Retraction'
+ WHEN 'goal' THEN NULL
+ WHEN 'memory_context' THEN NULL
+ WHEN 'file_event' THEN NULL
+ ELSE 'unmapped system entry kind=' || COALESCE(n.body ->> 'kind', '(none)') || ' at '
+ || e.session_id || '/' || e.seq
+ END
+ ELSE 'unmapped entry type=' || e.type || ' at ' || e.session_id || '/' || e.seq
+ END)::item_kind AS kind
+ OFFSET 0
+ ) k
+`;
+
+const HOIST = ` LEFT JOIN legacy_tape_steps l ON l.session_id = e.session_id AND l.seq = e.seq
+ CROSS JOIN LATERAL (
+ SELECT CASE WHEN k.kind IN ('UserMessage', 'SteeringMessage', 'AgentMessage', 'ContextCompaction') THEN 'text'
+ WHEN k.kind = 'Reasoning' THEN 'thinking'
+ WHEN k.kind = 'ToolResult' THEN 'result'
+ WHEN k.kind = 'Error' THEN 'message'
+ END AS text_key,
+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult', 'Reasoning', 'AgentMessage') THEN
+ COALESCE(CASE WHEN ${isType("step", "object")} THEN n.body -> 'step' END, l.step) END AS step_obj,
+ pg_input_is_valid(n.body ->> 'runId', 'uuid') AS run_is_uuid
+ ) t
+ CROSS JOIN LATERAL (
+ SELECT CASE WHEN jsonb_typeof(n.body -> t.text_key) = 'string' THEN t.text_key END AS text_key,
+ CASE WHEN jsonb_typeof(t.step_obj -> 'index') = 'number' THEN (t.step_obj ->> 'index')::int END AS step,
+ CASE WHEN t.step_obj IS NOT NULL
+ THEN COALESCE(CASE WHEN ${isType("wire", "object")} THEN n.body -> 'wire' END, l.wire, '{}'::jsonb)
+ || jsonb_build_object('message', t.step_obj - 'index')
+ WHEN NOT n.body ? 'wire' AND l.wire IS NOT NULL THEN l.wire
+ END AS wire
+ ) s
+ CROSS JOIN LATERAL (
+ SELECT CASE WHEN k.kind IN ${AUTHOR_KINDS_SQL} AND n.body ->> 'sourceRole' IS DISTINCT FROM 'agent' THEN
+ COALESCE(
+ CASE WHEN NOT n.body @> '{"overheard":true}' THEN
+ (SELECT min(p.principal_id) FROM participants p
+ WHERE p.session_id = e.session_id AND p.valid_from <= e.created_at
+ AND (p.valid_to IS NULL OR p.valid_to > e.created_at)
+ HAVING count(*) = 1) END,
+ CASE WHEN btrim(${str("name")}) <> '' THEN
+ (SELECT min(p.principal_id) FROM participants p
+ WHERE p.session_id = e.session_id
+ AND EXISTS (SELECT 1 FROM directory_members d
+ WHERE d.principal_id = p.principal_id
+ AND d.display_name_lc = lower(btrim(n.body ->> 'name')))
+ HAVING count(*) = 1) END)
+ END AS author,
+ CASE WHEN k.kind = 'Retraction' AND ${isType("ts", "string")} THEN
+ (SELECT max(u.seq) FROM session_entries u
+ WHERE u.session_id = e.session_id AND u.seq < e.seq AND u.type = 'user'
+ AND items_copy_payload(u.payload) ->> 'ts' = n.body ->> 'ts') END AS target_seq
+ OFFSET 0
+ ) a
+ CROSS JOIN LATERAL (
+ SELECT n.body - array_remove(ARRAY[
+ 'securityTainted',
+ CASE WHEN e.type = 'assistant' OR k.kind = 'Error' THEN 'deliveryKey' END,
+ s.text_key,
+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN 'tool' END,
+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN 'callId' END,
+ CASE WHEN k.kind IN ('Elicitation', 'ElicitationResponse') THEN 'requestId' END,
+ CASE WHEN k.kind = 'ToolResult' AND ${isType("isError", "boolean")} THEN 'isError' END,
+ CASE WHEN k.kind = 'UserMessage' AND ${isType("hidden", "boolean")} THEN 'hidden' END,
+ CASE WHEN k.kind = 'UserMessage' AND ${isType("ts", "string")} THEN 'ts' END,
+ CASE WHEN e.type = 'user' THEN 'steered' END,
+ CASE WHEN k.kind = 'ContextCompaction' AND ${isType("throughSeq", "number")} THEN 'throughSeq' END,
+ CASE WHEN k.kind = 'ElicitationResponse' AND ${isType("approved", "boolean")} THEN 'approved' END,
+ CASE WHEN e.type = 'system' AND n.body ->> 'kind' <> 'goal_cancelled' THEN 'kind' END,
+ CASE WHEN k.kind = 'Wake' THEN 'cause' END,
+ CASE WHEN t.run_is_uuid THEN 'runId' END,
+ CASE WHEN t.step_obj IS NOT NULL THEN 'step' END,
+ CASE WHEN s.wire IS NOT NULL THEN 'wire' END,
+ CASE WHEN s.text_key = 'result' AND ${isType("stdout", "string")}
+ AND strpos(n.body ->> 'result', n.body ->> 'stdout') > 0 THEN 'stdout' END,
+ CASE WHEN s.text_key = 'result' AND ${isType("stderr", "string")}
+ AND strpos(n.body ->> 'result', n.body ->> 'stderr') > 0 THEN 'stderr' END
+ ]::text[], NULL)
+ || CASE WHEN s.wire IS NOT NULL THEN jsonb_build_object('wire', s.wire) ELSE '{}'::jsonb END AS body
+ ) x
+`;
+
+const COLUMNS = `tenant_id, session_id, seq, turn_id, kind, author, call_id, elicit_id, wake_cause, error_code,
+ text, tool, step, variant, is_error, hidden, external_id, through_seq, target_seq, approved,
+ body, created_at`;
+
+const SELECT = `SELECT ${TENANT}, e.session_id, e.seq,
+ CASE WHEN t.run_is_uuid THEN (n.body ->> 'runId')::uuid END,
+ k.kind,
+ a.author,
+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') AND octet_length(${str("callId")}) <= 1024 AND ambiguous.call_id IS NULL
+ THEN NULLIF(${str("callId")}, '') END,
+ CASE WHEN k.kind IN ('Elicitation', 'ElicitationResponse') THEN
+ COALESCE(NULLIF(${str("requestId")}, ''), 'missing-elicit:' || e.seq) END,
+ CASE WHEN k.kind = 'Wake' THEN
+ CASE WHEN n.body ? 'cause' THEN (n.body ->> 'cause')::wake_cause ELSE 'system'::wake_cause END END,
+ CASE WHEN k.kind = 'Error' THEN 'turn_failure' END,
+ NULLIF(n.body ->> s.text_key, ''),
+ CASE WHEN k.kind IN ('ToolCall', 'ToolResult') THEN NULLIF(${str("tool")}, '') END,
+ s.step,
+ CASE e.type WHEN 'text' THEN 'narration' WHEN 'delivery' THEN 'files_delivered' END,
+ CASE WHEN k.kind = 'ToolResult' AND ${isType("isError", "boolean")} THEN (n.body -> 'isError')::boolean END,
+ CASE WHEN k.kind = 'UserMessage' AND ${isType("hidden", "boolean")} THEN (n.body -> 'hidden')::boolean END,
+ CASE WHEN k.kind = 'UserMessage' THEN NULLIF(${str("ts")}, '') END,
+ CASE WHEN k.kind = 'ContextCompaction' AND ${isType("throughSeq", "number")} THEN (n.body ->> 'throughSeq')::numeric::bigint END,
+ a.target_seq,
+ CASE WHEN k.kind = 'ElicitationResponse' AND ${isType("approved", "boolean")} THEN (n.body -> 'approved')::boolean END,
+ x.body,
+ to_timestamp(e.created_at / 1000.0)`;
+
+export function itemsCopySql(opts: ItemsCopyOptions = {}): string {
+ const from = opts.delta
+ ? ` FROM sessions sess
+ CROSS JOIN LATERAL (
+ SELECT COALESCE(max(i.seq), -1) AS seq FROM items i WHERE i.tenant_id = ${TENANT} AND i.session_id = sess.id
+ ) copied
+ CROSS JOIN LATERAL (
+ SELECT * FROM session_entries d WHERE d.session_id = sess.id AND d.seq > copied.seq OFFSET 0
+ ) e
+`
+ : ` FROM session_entries e
+`;
+ const bucket = opts.bucket ? `${bucketPredicate(opts.delta ? "sess.id" : "e.session_id", opts.bucket)}\n AND ` : "";
+ return insertRows(`WITH ambiguous_calls AS MATERIALIZED (${ambiguousCallsSql(false)})
+${SELECT}
+${from}${NORMALIZE}${KIND}${CALL_IDS}${HOIST} WHERE ${bucket}k.kind IS NOT NULL`);
+}
+
+function insertRows(rows: string): string {
+ return `WITH inserted AS (
+ INSERT INTO items(${COLUMNS}) ${rows} RETURNING kind,author
+ ) SELECT count(*)::int AS rows,
+ count(*) FILTER (WHERE kind IN ${AUTHOR_KINDS_SQL})::int AS author_eligible,
+ count(*) FILTER (WHERE kind IN ${AUTHOR_KINDS_SQL} AND author IS NULL)::int AS author_null
+ FROM inserted`;
+}
+
+export function itemsSelectedRowsSql(): string {
+ return `WITH ambiguous_calls AS MATERIALIZED (${ambiguousCallsSql(true)})
+ SELECT * FROM (
+ ${SELECT}
+ FROM pg_temp.items_copy_batch b
+ JOIN session_entries e ON (e.session_id,e.seq)=(b.session_id,b.seq)
+ ${NORMALIZE}${KIND}${CALL_IDS}${HOIST} WHERE k.kind IS NOT NULL
+ ) AS mapped(${COLUMNS})`;
+}
+
+export function itemsCopySelectedSql(): string {
+ return insertRows(itemsSelectedRowsSql());
+}
diff --git a/src/sessions/items-schema.ts b/src/sessions/items-schema.ts
new file mode 100644
index 000000000..5d6f07bdd
--- /dev/null
+++ b/src/sessions/items-schema.ts
@@ -0,0 +1,147 @@
+import { itemsCopySql, ITEMS_COPY_REPAIR_SQL } from "./items-copy.ts";
+
+const ITEM_KINDS = [
+ "UserMessage",
+ "SteeringMessage",
+ "AgentMessage",
+ "Reasoning",
+ "ToolCall",
+ "ToolResult",
+ "Elicitation",
+ "ElicitationResponse",
+ "Wake",
+ "Error",
+ "ContextCompaction",
+ "Retraction",
+ "Stop",
+ "RuntimeChange",
+] as const;
+
+type ItemKind = (typeof ITEM_KINDS)[number];
+
+const WAKE_CAUSES = ["cron", "webhook", "child_done", "system"] as const;
+
+const sqlList = (values: readonly string[]): string => values.map((value) => `'${value}'`).join(", ");
+
+const AUTHOR_KINDS = ["UserMessage", "SteeringMessage", "ElicitationResponse", "Stop", "RuntimeChange"] as const;
+
+const HOISTED_COLUMN_KINDS: ReadonlyArray<[column: string, kinds: readonly ItemKind[]]> = [
+ ["text", ["UserMessage", "SteeringMessage", "AgentMessage", "Reasoning", "ToolResult", "Error", "ContextCompaction"]],
+ ["tool", ["ToolCall", "ToolResult"]],
+ ["step", ["ToolCall", "ToolResult", "Reasoning", "AgentMessage"]],
+ ["variant", ["AgentMessage"]],
+ ["is_error", ["ToolResult"]],
+ ["hidden", ["UserMessage"]],
+ ["external_id", ["UserMessage"]],
+ ["through_seq", ["ContextCompaction"]],
+ ["target_seq", ["Retraction"]],
+ ["approved", ["ElicitationResponse"]],
+ ["blob_sha", ["ToolResult"]],
+];
+
+const hoistedChecks = HOISTED_COLUMN_KINDS.map(
+ ([column, kinds]) => `CONSTRAINT items_${column}_kind CHECK (${column} IS NULL OR kind IN (${sqlList(kinds)}))`,
+).join(",\n ");
+
+export const ITEMS_TABLES_SQL: readonly string[] = [
+ `CREATE TABLE IF NOT EXISTS qm_tenant(id UUID PRIMARY KEY)`,
+ `CREATE UNIQUE INDEX IF NOT EXISTS qm_tenant_singleton ON qm_tenant((true))`,
+ `INSERT INTO qm_tenant(id) SELECT gen_random_uuid() WHERE NOT EXISTS (SELECT 1 FROM qm_tenant)`,
+ `CREATE TABLE IF NOT EXISTS qm_settings(
+ key TEXT PRIMARY KEY,
+ value JSONB NOT NULL,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+ )`,
+ `DO $item_kind$ BEGIN
+ CREATE TYPE item_kind AS ENUM (${sqlList(ITEM_KINDS)});
+ EXCEPTION WHEN duplicate_object THEN NULL; END $item_kind$`,
+ `DO $wake_cause$ BEGIN
+ CREATE TYPE wake_cause AS ENUM (${sqlList(WAKE_CAUSES)});
+ EXCEPTION WHEN duplicate_object THEN NULL; END $wake_cause$`,
+ `CREATE TABLE IF NOT EXISTS blobs(
+ tenant_id UUID NOT NULL,
+ sha256 BYTEA NOT NULL CHECK (length(sha256) = 32),
+ size BIGINT NOT NULL CHECK (size >= 0),
+ mime TEXT,
+ storage_key TEXT NOT NULL CHECK (storage_key <> ''),
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ expires_at TIMESTAMPTZ,
+ PRIMARY KEY (tenant_id, sha256)
+ )`,
+ `CREATE TABLE IF NOT EXISTS items(
+ tenant_id UUID NOT NULL,
+ session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+ seq BIGINT NOT NULL CHECK (seq >= 0),
+ turn_id UUID,
+ kind item_kind NOT NULL,
+ author TEXT CHECK (author <> ''),
+ call_id TEXT CHECK (call_id <> ''),
+ elicit_id TEXT CHECK (elicit_id <> ''),
+ wake_cause wake_cause,
+ error_code TEXT CHECK (error_code <> ''),
+ text TEXT CHECK (text <> ''),
+ tool TEXT CHECK (tool <> ''),
+ step INT CHECK (step >= 0),
+ variant TEXT CHECK (variant IN ('narration', 'files_delivered')),
+ is_error BOOLEAN,
+ hidden BOOLEAN,
+ external_id TEXT CHECK (external_id <> ''),
+ through_seq BIGINT,
+ target_seq BIGINT,
+ approved BOOLEAN,
+ blob_sha BYTEA,
+ search TSVECTOR GENERATED ALWAYS AS (to_tsvector('simple', coalesce(text, ''))) STORED,
+ body JSONB NOT NULL CHECK (jsonb_typeof(body) = 'object'),
+ created_at TIMESTAMPTZ NOT NULL,
+ PRIMARY KEY (tenant_id, session_id, seq),
+ CONSTRAINT items_call_id_kind CHECK (call_id IS NULL OR kind IN ('ToolCall', 'ToolResult')),
+ CONSTRAINT items_elicit_id_kind CHECK ((elicit_id IS NOT NULL) = (kind IN ('Elicitation', 'ElicitationResponse'))),
+ CONSTRAINT items_wake_cause_kind CHECK ((wake_cause IS NOT NULL) = (kind = 'Wake')),
+ CONSTRAINT items_error_code_kind CHECK ((error_code IS NOT NULL) = (kind = 'Error')),
+ CONSTRAINT items_author_kind CHECK (author IS NULL OR kind IN (${sqlList(AUTHOR_KINDS)})),
+ ${hoistedChecks},
+ CONSTRAINT items_blob FOREIGN KEY (tenant_id, blob_sha) REFERENCES blobs(tenant_id, sha256)
+ )`,
+];
+
+export interface ItemIndex {
+ name: string;
+ unique?: boolean;
+ on: string;
+}
+
+export const ITEM_INDEXES: readonly ItemIndex[] = [
+ {
+ name: "items_one_call_per_kind",
+ unique: true,
+ on: `items(tenant_id, session_id, kind, call_id) WHERE call_id IS NOT NULL`,
+ },
+ { name: "items_by_turn", on: `items(tenant_id, turn_id, seq) WHERE turn_id IS NOT NULL` },
+ { name: "items_by_elicit", on: `items(tenant_id, session_id, elicit_id) WHERE elicit_id IS NOT NULL` },
+ {
+ name: "items_by_external_id",
+ on: `items(tenant_id, session_id, external_id) WHERE external_id IS NOT NULL`,
+ },
+ { name: "items_session_kind_created", on: `items(tenant_id, session_id, kind, created_at)` },
+ { name: "items_search", on: `items USING gin (tenant_id, session_id, search)` },
+];
+
+export function itemIndexSql(index: ItemIndex, opts: { concurrently?: boolean } = {}): string {
+ return `CREATE ${index.unique ? "UNIQUE " : ""}INDEX ${opts.concurrently ? "CONCURRENTLY " : ""}IF NOT EXISTS ${index.name} ON ${index.on}`;
+}
+
+export const itemsMigration = {
+ id: "sessions/store/0035-items",
+ statements: [
+ `DO $operator$ BEGIN
+ IF to_regclass('items_copy_ops.state') IS NOT NULL THEN
+ RAISE EXCEPTION 'An items copy is staged; complete the coordinated cutover before applying this migration';
+ END IF;
+ END $operator$`,
+ ...ITEMS_TABLES_SQL,
+ ...ITEMS_COPY_REPAIR_SQL,
+ `SET LOCAL statement_timeout = 0`,
+ itemsCopySql({ delta: true }),
+ ...ITEM_INDEXES.map((index) => itemIndexSql(index)),
+ ],
+};
diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts
index 69c578547..695dc7141 100644
--- a/src/sessions/postgres-session-store.ts
+++ b/src/sessions/postgres-session-store.ts
@@ -1,3 +1,4 @@
+import { DIRECTORY_MIGRATIONS } from "../directory/postgres-directory-store.ts";
import { randomUUID } from "node:crypto";
import {
createPgPool,
@@ -40,6 +41,7 @@ import type {
StoreOptions,
} from "./session-store.ts";
import { tsPrefixQuery } from "./entry-search.ts";
+import { itemsMigration } from "./items-schema.ts";
import {
DROP_SESSION_TAPE_SQL,
LEGACY_TAPE_STEPS_SIZE_GUARD_SQL,
@@ -345,6 +347,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
const { pool, q } = createPgPool(
connectionString,
[
+ ...DIRECTORY_MIGRATIONS,
{
id: "sessions/store/0001",
expectedChecksum: "cf56c9f6488806677a229698193ef6dd7cb74300c343bdb1930c17702c5ccf2f",
@@ -880,6 +883,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Store
id: "sessions/store/0033-client-tools",
statements: [`ALTER TABLE sessions ADD COLUMN IF NOT EXISTS client_tools JSONB`],
},
+ itemsMigration,
],
[
{
diff --git a/test/loop-fire.test.ts b/test/loop-fire.test.ts
index 7bdcb6b72..0256c91d9 100644
--- a/test/loop-fire.test.ts
+++ b/test/loop-fire.test.ts
@@ -1013,16 +1013,17 @@ test("a failed followup rejects so the composer can retain uploaded attachments"
});
test("triage groups a flood read-only so only the representative is worked, with the rest as evidence", async () => {
- const s = service((req) => {
+ const s = service(async (req) => {
const text = req.text ?? "";
if (text.startsWith("[Loop triage]")) {
- const ids = [...text.matchAll(/"id":"([^"]+)"/g)].map((match) => match[1]!);
+ const open = await s.items.byLoop(loop.id);
+ const representative = open.find((item) => item.sourceKey === "SENTRY-1")!;
return `\`\`\`json\n${JSON.stringify({
- items: ids.map((id, index) => ({
- id,
- priority: index === 0 ? "urgent" : "low",
+ items: open.map((item) => ({
+ id: item.id,
+ priority: item.id === representative.id ? "urgent" : "low",
reason: "checkout is down",
- ...(index > 0 ? { groupWith: ids[0] } : {}),
+ ...(item.id !== representative.id ? { groupWith: representative.id } : {}),
})),
})}\n\`\`\``;
}
diff --git a/test/postgres-items-copy.test.ts b/test/postgres-items-copy.test.ts
new file mode 100644
index 000000000..4cdfd5294
--- /dev/null
+++ b/test/postgres-items-copy.test.ts
@@ -0,0 +1,575 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { randomUUID } from "node:crypto";
+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";
+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";
+import { itemsCopySql, itemsCopySelectedSql, itemsSelectedRowsSql } from "../src/sessions/items-copy.ts";
+import { ITEM_INDEXES } from "../src/sessions/items-schema.ts";
+import { scopeId } from "../src/types.ts";
+
+type Pool = import("pg").Pool;
+
+const URL = process.env.DATABASE_URL;
+const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the items copy test";
+const T0 = 1_700_000_000_000;
+
+const ITEMS_MIGRATION = "sessions/store/0035-items";
+
+async function withSchema(
+ fn: (raw: Pool, url: string) => Promise<void>,
+ opts: { beforeItems?: (raw: Pool) => Promise<void> } = {},
+): Promise<void> {
+ const pg = (await import("pg")).default;
+ const admin = new pg.Pool({ connectionString: URL });
+ const schema = `items_copy_${randomUUID().replaceAll("-", "")}`;
+ await admin.query(`CREATE SCHEMA ${schema}`);
+ const url = new globalThis.URL(URL!);
+ url.searchParams.set("options", `-c search_path=${schema}`);
+ const raw = new pg.Pool({ connectionString: url.toString() });
+ try {
+ createPostgresSessionStore(url.toString());
+ createPostgresDirectoryStore(url.toString());
+ const migrations = registeredPgMigrations(url.toString());
+ if (opts.beforeItems) {
+ await applyPgMigrations(
+ raw,
+ migrations.filter((migration) => migration.id !== ITEMS_MIGRATION),
+ );
+ await opts.beforeItems(raw);
+ }
+ await applyPgMigrations(raw, migrations);
+ await fn(raw, url.toString());
+ } finally {
+ await raw.end();
+ await admin.query(`DROP SCHEMA ${schema} CASCADE`);
+ await admin.end();
+ }
+}
+
+const scope = scopeId("personal", "alice");
+
+async function seed(raw: Pool, id: string, entries: Array<[string, unknown]>, from = 0) {
+ await raw.query(
+ "INSERT INTO sessions(id, type, scope_id, thread_ref, created_at) VALUES ($1, 'dm', $2, $3, 1) ON CONFLICT DO NOTHING",
+ [id, scope, `web:${id}`],
+ );
+ for (const [i, [type, payload]] of entries.entries())
+ await raw.query(
+ "INSERT INTO session_entries(session_id, seq, parent_seq, type, payload, scope_label, created_at) VALUES ($1, $2, NULL, $3, $4, $5, $6)",
+ [id, from + i, type, JSON.stringify(payload), scope, T0 + from + i],
+ );
+}
+
+async function copyAll(raw: Pool, buckets = 4, delta = false): Promise<number> {
+ let total = 0;
+ for (let index = 0; index < buckets; index++)
+ total += (await raw.query(itemsCopySql({ delta, bucket: { index, count: buckets } }))).rows[0].rows;
+ return total;
+}
+
+const items = async (raw: Pool, sessionId: string) =>
+ (
+ await raw.query(
+ `SELECT seq::int, turn_id, kind, author, call_id, elicit_id, wake_cause, error_code, text, tool, step,
+ variant, is_error, hidden, external_id, through_seq::int, target_seq::int, approved, body,
+ (extract(epoch FROM created_at) * 1000)::bigint::text AS ms
+ FROM items WHERE session_id = $1 ORDER BY seq`,
+ [sessionId],
+ )
+ ).rows as Array<Record<string, unknown> & { seq: number; body: Record<string, unknown> }>;
+
+test("items copy maps every legacy type, hoists fields into columns and strips them from body", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const run = randomUUID();
+ await seed(raw, "s1", [
+ ["soul", { text: "persona" }],
+ ["user", { text: "hi", runId: run, securityTainted: true, ts: "1.1", hidden: true, name: "Al" }],
+ ["user", { text: "also", steered: true, ts: "1.5" }],
+ ["text_start", {}],
+ ["text", { text: "narrating" }],
+ ["thinking", { thinking: "hmm", thinkingSignature: "sig" }],
+ ["tool_call", { tool: "execute", callId: "c1", args: { cmd: "ls" } }],
+ [
+ "tool_result",
+ {
+ tool: "execute",
+ callId: "c1",
+ result: "out: hello\nerr: boom",
+ stdout: "hello",
+ stderr: "nope",
+ isError: true,
+ },
+ ],
+ ["tool_call", { tool: "execute" }],
+ ["assistant", { text: "done", deliveryKey: "k1", runId: "not-a-uuid", via: "cron" }],
+ ["delivery", { files: ["a.txt"] }],
+ ["approval_request", { requestId: "r1", command: "rm" }],
+ ["approval_resolved", { approved: true }],
+ ["system", { kind: "context_summary", text: "summary", throughSeq: 3 }],
+ ["system", { kind: "turn_failure", message: "failed", deliveryKey: "k2", runId: run }],
+ ["system", { kind: "wake", text: "follow-up" }],
+ ["system", { kind: "wake", cause: "cron", text: "fire" }],
+ ["system", { kind: "message_revision", action: "deleted", ts: "1.1" }],
+ ["system", { kind: "goal", goal: {} }],
+ ["system", { kind: "memory_context", snapshot: {} }],
+ ["system", { kind: "file_event", direction: "in" }],
+ ["user", "just a string"],
+ ["user", { type: "x", $type: "y", $blobs: [], text: "t" }],
+ ["assistant", 5],
+ ["tool_result", { callId: "", result: { ok: true }, stdout: "kept" }],
+ ["user", { text: "" }],
+ ]);
+ await raw.query(
+ `INSERT INTO legacy_tape_steps(session_id, seq, step, wire)
+ VALUES ('s1', 6, '{"id":"tape:1","model":"m","index":2}', '{"x":1}'), ('s1', 5, '{"id":"tape:1","index":0}', NULL)`,
+ );
+
+ assert.equal(await copyAll(raw), 21);
+ const rows = await items(raw, "s1");
+ const at = (seq: number) => rows.find((row) => row.seq === seq)!;
+ assert.deepEqual(
+ rows.map((row) => [row.seq, row.kind]),
+ [
+ [1, "UserMessage"],
+ [2, "SteeringMessage"],
+ [4, "AgentMessage"],
+ [5, "Reasoning"],
+ [6, "ToolCall"],
+ [7, "ToolResult"],
+ [8, "ToolCall"],
+ [9, "AgentMessage"],
+ [10, "AgentMessage"],
+ [11, "Elicitation"],
+ [12, "ElicitationResponse"],
+ [13, "ContextCompaction"],
+ [14, "Error"],
+ [15, "Wake"],
+ [16, "Wake"],
+ [17, "Retraction"],
+ [21, "UserMessage"],
+ [22, "UserMessage"],
+ [23, "AgentMessage"],
+ [24, "ToolResult"],
+ [25, "UserMessage"],
+ ],
+ );
+ assert.equal(at(1).ms, String(T0 + 1));
+ assert.deepEqual(
+ [at(1).turn_id, at(1).text, at(1).hidden, at(1).external_id, at(1).body],
+ [run, "hi", true, "1.1", { name: "Al" }],
+ );
+ assert.deepEqual([at(2).text, at(2).external_id, at(2).body], ["also", null, { ts: "1.5" }]);
+ assert.deepEqual([at(4).text, at(4).variant, at(4).body], ["narrating", "narration", {}]);
+ assert.deepEqual(
+ [at(5).text, at(5).step, at(5).body],
+ ["hmm", 0, { thinkingSignature: "sig", wire: { message: { id: "tape:1" } } }],
+ );
+ assert.deepEqual(
+ [at(6).call_id, at(6).tool, at(6).step, at(6).body],
+ ["c1", "execute", 2, { args: { cmd: "ls" }, wire: { x: 1, message: { id: "tape:1", model: "m" } } }],
+ );
+ assert.deepEqual(
+ [at(7).call_id, at(7).tool, at(7).text, at(7).is_error, at(7).body],
+ ["c1", "execute", "out: hello\nerr: boom", true, { stderr: "nope" }],
+ );
+ assert.equal(at(8).call_id, null);
+ assert.deepEqual(
+ [at(9).turn_id, at(9).text, at(9).variant, at(9).body],
+ [null, "done", null, { runId: "not-a-uuid", via: "cron" }],
+ );
+ assert.deepEqual([at(10).variant, at(10).text, at(10).body], ["files_delivered", null, { files: ["a.txt"] }]);
+ assert.deepEqual([at(11).elicit_id, at(11).body], ["r1", { command: "rm" }]);
+ assert.deepEqual([at(12).elicit_id, at(12).approved, at(12).body], ["missing-elicit:12", true, {}]);
+ assert.deepEqual([at(13).text, at(13).through_seq, at(13).body], ["summary", 3, {}]);
+ assert.deepEqual(
+ [at(14).error_code, at(14).text, at(14).turn_id, at(14).body],
+ ["turn_failure", "failed", run, {}],
+ );
+ assert.deepEqual([at(15).wake_cause, at(15).body], ["system", { text: "follow-up" }]);
+ assert.deepEqual([at(16).wake_cause, at(16).body], ["cron", { text: "fire" }]);
+ assert.deepEqual([at(17).target_seq, at(17).body], [1, { action: "deleted", ts: "1.1" }]);
+ assert.deepEqual([at(21).text, at(21).body], ["just a string", {}]);
+ assert.deepEqual([at(22).text, at(22).body], ["t", { type: "x", $type: "y", $blobs: [] }]);
+ assert.deepEqual(at(23).body, { value: 5 });
+ assert.deepEqual(
+ [at(24).call_id, at(24).text, at(24).body],
+ [null, null, { result: { ok: true }, stdout: "kept" }],
+ );
+ assert.deepEqual([at(25).text, at(25).body], [null, {}]);
+ });
+});
+
+test("items copy deduces legacy authors from participants and directory names", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const join = (session: string, principal: string, from: number, to: number | null) =>
+ raw.query("INSERT INTO participants(session_id, principal_id, valid_from, valid_to) VALUES ($1, $2, $3, $4)", [
+ session,
+ principal,
+ from,
+ to,
+ ]);
+ await raw.query(
+ `INSERT INTO directory_members(org_id, principal_id, display_name, display_name_lc, type)
+ VALUES ('o', 'u:ann', 'Ann Lee', 'ann lee', 'internal'), ('o', 'u:bob', 'Bob', 'bob', 'internal')`,
+ );
+ await seed(raw, "solo", [
+ ["user", { text: "before ann joined" }],
+ ["user", { text: "ann alone" }],
+ ["user", { text: "both here", name: "bob" }],
+ ["user", { text: "both here, nameless" }],
+ ["user", { text: "overheard", overheard: true, ts: "9.1", name: "Ann Lee" }],
+ ["user", { text: "agent echo", overheard: true, ts: "9.2", sourceRole: "agent", name: "Ann Lee" }],
+ ["approval_resolved", { requestId: "r", approved: false }],
+ ]);
+ await join("solo", "u:ann", T0 + 1, null);
+ await join("solo", "u:bob", T0 + 2, T0 + 6);
+ const counts = (await raw.query(itemsCopySql())).rows[0];
+ assert.deepEqual(counts, { rows: 7, author_eligible: 7, author_null: 3 });
+ assert.deepEqual(
+ (await items(raw, "solo")).map((row) => [row.seq, row.author]),
+ [
+ [0, null],
+ [1, "u:ann"],
+ [2, "u:bob"],
+ [3, null],
+ [4, "u:ann"],
+ [5, null],
+ [6, "u:ann"],
+ ],
+ );
+ });
+});
+
+test("items CHECKs tie each hoisted column to its kinds and search indexes the text", { skip }, async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "c", []);
+ const insert = (cols: string, values: string) =>
+ raw.query(
+ `INSERT INTO items(tenant_id, session_id, seq, body, created_at, ${cols})
+ SELECT id, 'c', (SELECT count(*) FROM items), '{}', now(), ${values} FROM qm_tenant`,
+ );
+ await insert("kind, text", "'UserMessage', 'the quick brown fox'");
+ await insert("kind, call_id, tool, is_error", "'ToolResult', 'c1', 'execute', false");
+ for (const [cols, values, constraint] of [
+ ["kind, tool", "'UserMessage', 'execute'", /items_tool_kind/],
+ ["kind, variant", "'AgentMessage', 'reply'", /variant_check/],
+ ["kind, text", "'UserMessage', ''", /text_check/],
+ ["kind, hidden", "'SteeringMessage', true", /items_hidden_kind/],
+ ["kind, external_id", "'AgentMessage', '1.1'", /items_external_id_kind/],
+ ["kind, through_seq", "'Retraction', 1", /items_through_seq_kind/],
+ ["kind, target_seq", "'ContextCompaction', 1", /items_target_seq_kind/],
+ ["kind, approved", "'Elicitation', true", /items_approved_kind/],
+ ["kind, author", "'AgentMessage', 'u:ann'", /items_author_kind/],
+ ["kind, call_id, blob_sha", "'ToolResult', 'c2', sha256('x')", /items_blob/],
+ ] as const)
+ await assert.rejects(insert(cols, values), constraint);
+ await insert("kind, author", "'Stop', 'u:ann'");
+ await insert("kind, author", "'RuntimeChange', 'u:ann'");
+ const hits = await raw.query(
+ "SELECT seq::int FROM items WHERE session_id = 'c' AND search @@ to_tsquery('simple', 'brown & fox')",
+ );
+ assert.deepEqual(hits.rows, [{ seq: 0 }]);
+ });
+});
+
+test("items copy fails loudly on an unmapped entry type or system kind", { skip }, async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "ok", [["user", { text: "fine" }]]);
+ await seed(raw, "bad", [["system", { kind: "mystery" }]]);
+ await assert.rejects(copyAll(raw, 1), /unmapped system entry kind=mystery at bad\/0/);
+ await raw.query("DELETE FROM session_entries WHERE session_id = 'bad'");
+ await seed(raw, "bad2", [["weird", { text: "?" }]]);
+ await assert.rejects(copyAll(raw, 1), /unmapped entry type=weird at bad2\/0/);
+ assert.equal((await raw.query("SELECT count(*)::int AS n FROM items")).rows[0].n, 0);
+ });
+});
+
+test("bucketed bulk copy partitions sessions and delta copies only new entries", { skip }, async () => {
+ await withSchema(async (raw) => {
+ for (let i = 0; i < 12; i++)
+ await seed(raw, `s${i}`, [
+ ["user", { text: `q${i}` }],
+ ["assistant", { text: `a${i}` }],
+ ]);
+ assert.equal(await copyAll(raw), 24);
+ assert.equal(await copyAll(raw, 4, true), 0);
+ await seed(
+ raw,
+ "s3",
+ [
+ ["user", { text: "later" }],
+ ["system", { kind: "goal" }],
+ ["thinking", { thinking: "x" }],
+ ],
+ 2,
+ );
+ await seed(raw, "fresh", [["user", { text: "new session" }]]);
+ assert.equal(await copyAll(raw, 1, true), 3);
+ assert.deepEqual(
+ (await items(raw, "s3")).map((row) => [row.seq, row.kind]),
+ [
+ [0, "UserMessage"],
+ [1, "AgentMessage"],
+ [2, "UserMessage"],
+ [4, "Reasoning"],
+ ],
+ );
+ assert.equal(await copyAll(raw, 1, true), 0);
+ assert.throws(() => itemsCopySql({ bucket: { index: 4, count: 4 } }), /bucket 4 of 4 is invalid/);
+ });
+});
+
+test("the startup migration copies existing history and builds the indexes", { skip }, async () => {
+ await withSchema(
+ async (raw) => {
+ assert.deepEqual(
+ (await items(raw, "old")).map((row) => [row.seq, row.kind, row.text]),
+ [
+ [0, "UserMessage", "before items existed"],
+ [1, "ToolCall", null],
+ ],
+ );
+ const built = await raw.query(
+ `SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
+ WHERE i.indrelid = 'items'::regclass AND i.indisvalid AND c.relname = ANY($1)`,
+ [ITEM_INDEXES.map((index) => index.name)],
+ );
+ assert.equal(built.rowCount, ITEM_INDEXES.length);
+ },
+ {
+ beforeItems: (raw) =>
+ seed(raw, "old", [
+ ["user", { text: "before items existed" }],
+ ["tool_call", { tool: "deliver", callId: "c1" }],
+ ]),
+ },
+ );
+});
+
+test(
+ "items copy repairs unsupported JSON escapes, preserves literals and rejects other broken payloads",
+ { skip },
+ async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "repair", []);
+ const payloads = [
+ String.raw`{"text":"nul\u0000end"}`,
+ String.raw`{"text":"high\ud800end"}`,
+ String.raw`{"text":"low\udc00end"}`,
+ String.raw`{"text":"pair\ud83d\ude00"}`,
+ String.raw`{"text":"literal\\u0000"}`,
+ null,
+ "7",
+ "true",
+ ];
+ for (const [seq, payload] of payloads.entries())
+ await raw.query(
+ "INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES('repair',$1,'user',$2,'personal:test',1)",
+ [seq, payload],
+ );
+ assert.equal(await copyAll(raw, 1), 8);
+ const result = await items(raw, "repair");
+ assert.deepEqual(
+ result.slice(0, 5).map((r) => r.text),
+ ["nulend", "high�end", "low�end", "pair😀", String.raw`literal\u0000`],
+ );
+ assert.deepEqual(
+ result.slice(5).map((r) => r.body),
+ [{}, { value: 7 }, { value: true }],
+ );
+ await raw.query("UPDATE session_entries SET payload='not json' WHERE seq=0");
+ await raw.query("DELETE FROM items");
+ await assert.rejects(copyAll(raw, 1), /invalid input syntax/);
+ assert.equal((await raw.query("SELECT count(*)::int AS n FROM items")).rows[0].n, 0);
+ });
+ },
+);
+
+test("selected-key copying uses the same mapping on TEXT and JSONB without copying other rows", { skip }, async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "selected", [
+ ["user", { text: "one" }],
+ ["assistant", { text: "two" }],
+ ["soul", {}],
+ ]);
+ const client = await raw.connect();
+ try {
+ await client.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");
+ await client.query("INSERT INTO items_copy_batch VALUES('selected',1),('selected',2)");
+ assert.equal((await client.query(itemsCopySelectedSql())).rows[0].rows, 1);
+ assert.deepEqual(
+ (await items(raw, "selected")).map((r) => r.text),
+ ["two"],
+ );
+ const expected = (await client.query(itemsSelectedRowsSql())).rows[0];
+ assert.equal(expected.kind, "AgentMessage");
+ assert.equal(expected.text, "two");
+ await client.query(
+ "CREATE TEMP TABLE source_rows AS SELECT session_id,seq,type,payload,created_at FROM session_entries",
+ );
+ await client.query("CREATE TEMP TABLE session_entries AS SELECT * FROM source_rows WITH NO DATA");
+ await client.query("ALTER TABLE pg_temp.session_entries ALTER COLUMN payload TYPE jsonb USING payload::jsonb");
+ await client.query(
+ "INSERT INTO pg_temp.session_entries SELECT session_id,seq,type,payload::jsonb,created_at FROM source_rows",
+ );
+ await client.query("DELETE FROM items");
+ assert.equal((await client.query(itemsCopySelectedSql())).rows[0].rows, 1);
+ assert.equal((await client.query(itemsSelectedRowsSql())).rows[0].text, "two");
+ } finally {
+ client.release();
+ }
+ });
+});
+
+test("items copy preserves missing call IDs as NULL and still constrains present IDs", { skip }, async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "nullable-calls", [
+ ["tool_call", { tool: "execute", command: "echo hello" }],
+ ["tool_result", { tool: "execute", result: "hello" }],
+ ["tool_call", { tool: "execute", callId: "" }],
+ ["tool_result", { tool: "execute", callId: null }],
+ ["tool_call", { tool: "execute", callId: "real-call" }],
+ ["tool_result", { tool: "execute", callId: "real-call" }],
+ ]);
+ await copyAll(raw, 1);
+ const rows = await items(raw, "nullable-calls");
+ assert.deepEqual(
+ rows.map((row) => row.call_id),
+ [null, null, null, null, "real-call", "real-call"],
+ );
+ assert.ok(rows.every((row) => !("callId" in row.body)));
+ await assert.rejects(
+ raw.query("UPDATE items SET call_id='' WHERE session_id='nullable-calls' AND seq=0"),
+ /check constraint/,
+ );
+ await assert.rejects(
+ raw.query("UPDATE items SET call_id='real-call' WHERE session_id='nullable-calls' AND seq=0"),
+ /unique constraint/,
+ );
+ await assert.rejects(
+ raw.query("UPDATE items SET kind='UserMessage',tool=NULL WHERE session_id='nullable-calls' AND seq=4"),
+ /items_call_id_kind/,
+ );
+ });
+});
+
+test("payload parsing keeps ordinary JSON fast-path and Unicode repair semantics", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const valid: Array<[string, unknown]> = [
+ [
+ String.raw`{"text":"ordinary \"quoted\" text","values":[1,true,null]}`,
+ { text: 'ordinary "quoted" text', values: [1, true, null] },
+ ],
+ [String.raw`{"text":"\u0041"}`, { text: "A" }],
+ [String.raw`{"text":"before\u0000after"}`, { text: "beforeafter" }],
+ [String.raw`{"text":"before\ud800after"}`, { text: "before�after" }],
+ [String.raw`{"text":"literal\\u0000"}`, { text: String.raw`literal\u0000` }],
+ ];
+ for (const [input, expected] of valid)
+ assert.deepEqual(
+ (await raw.query("SELECT items_copy_payload($1::text) AS parsed", [input])).rows[0].parsed,
+ expected,
+ );
+ for (const input of [String.raw`{"text":"\q"}`, '{"text":"unfinished"', String.raw`{"text":"\u0000","bad":}`])
+ await assert.rejects(raw.query("SELECT items_copy_payload($1::text)", [input]), /invalid input syntax/);
+ });
+});
+
+test("items copy preserves stopped-turn records", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const runId = "11111111-1111-4111-8111-111111111111";
+ await seed(raw, "stopped", [["system", { kind: "turn_stopped", runId, reason: "user", text: "Stopped" }]]);
+ await raw.query("INSERT INTO participants(session_id, principal_id, valid_from) VALUES ('stopped', 'u:ann', 0)");
+ await raw.query(itemsCopySql());
+ const [row] = await items(raw, "stopped");
+ assert.ok(row);
+ assert.equal(row.kind, "Stop");
+ assert.equal(row.turn_id, runId);
+ assert.equal(row.author, "u:ann");
+ assert.deepEqual(row.body, { reason: "user", text: "Stopped" });
+ });
+});
+
+test("items copy maps goal cancellation notices to system wakes", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const body = { kind: "goal_cancelled", objective: "A sample goal", cancelledBy: "u:ann" };
+ await seed(raw, "cancelled-goal", [["system", body]]);
+ await raw.query(itemsCopySql());
+ const [row] = await items(raw, "cancelled-goal");
+ assert.ok(row);
+ assert.equal(row.kind, "Wake");
+ assert.equal(row.wake_cause, "system");
+ assert.equal(row.author, null);
+ assert.deepEqual(row.body, body);
+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");
+ await raw.query("INSERT INTO items_copy_batch VALUES ('cancelled-goal', 0)");
+ const selected = (await raw.query(itemsSelectedRowsSql())).rows[0];
+ assert.equal(selected.kind, "Wake");
+ assert.equal(selected.wake_cause, "system");
+ assert.deepEqual(selected.body, body);
+ });
+});
+
+test("oversized legacy call identifiers become NULL without dropping history", { skip }, async () => {
+ await withSchema(async (raw) => {
+ const callId = "x".repeat(1025);
+ const boundary = "x".repeat(1024);
+ await seed(raw, "long-calls", [
+ ["tool_call", { tool: "execute", callId, command: "echo preserved" }],
+ ["tool_result", { tool: "execute", callId, result: "preserved output" }],
+ ["tool_call", { tool: "execute", callId: boundary }],
+ ["tool_result", { tool: "execute", callId: boundary, result: "boundary output" }],
+ ["tool_call", { tool: "execute", callId: "é".repeat(513) }],
+ ]);
+ await raw.query(itemsCopySql());
+ const rows = await items(raw, "long-calls");
+ assert.deepEqual(
+ rows.map((row) => row.call_id),
+ [null, null, boundary, boundary, null],
+ );
+ assert.equal(rows[0]!.body.command, "echo preserved");
+ assert.equal(rows[1]!.text, "preserved output");
+ assert.equal(rows[3]!.text, "boundary output");
+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");
+ await raw.query("INSERT INTO items_copy_batch SELECT session_id,seq FROM session_entries");
+ const selected = (await raw.query(itemsSelectedRowsSql())).rows;
+ assert.deepEqual(
+ selected.map((row) => row.call_id),
+ [null, null, boundary, boundary, null],
+ );
+ });
+});
+
+test("ambiguous legacy IDs are cleared across calls and results, including across batches", { skip }, async () => {
+ await withSchema(async (raw) => {
+ await seed(raw, "ambiguous", [
+ ["tool_call", { tool: "execute", callId: "repeated-result", command: "preserved" }],
+ ["tool_result", { tool: "execute", callId: "repeated-result", result: "first" }],
+ ["tool_result", { tool: "execute", callId: "repeated-result", result: "second" }],
+ ["tool_call", { tool: "execute", callId: "repeated-call" }],
+ ["tool_call", { tool: "execute", callId: "repeated-call" }],
+ ["tool_result", { tool: "execute", callId: "repeated-call", result: "third" }],
+ ["tool_call", { tool: "execute", callId: "valid-pair" }],
+ ["tool_result", { tool: "execute", callId: "valid-pair", result: "valid" }],
+ ]);
+ await seed(raw, "other-session", [["tool_call", { tool: "execute", callId: "repeated-call" }]]);
+ await raw.query("CREATE TEMP TABLE items_copy_batch(session_id text,seq int,PRIMARY KEY(session_id,seq))");
+ await raw.query("INSERT INTO items_copy_batch VALUES ('ambiguous',0),('ambiguous',1)");
+ await raw.query(itemsCopySelectedSql());
+ assert.deepEqual(
+ (await items(raw, "ambiguous")).map((row) => row.call_id),
+ [null, null],
+ );
+ await raw.query("TRUNCATE items");
+ await raw.query(itemsCopySql());
+ const rows = await items(raw, "ambiguous");
+ assert.deepEqual(
+ rows.map((row) => row.call_id),
+ [null, null, null, null, null, null, "valid-pair", "valid-pair"],
+ );
+ assert.equal(rows[0]!.body.command, "preserved");
+ assert.deepEqual([rows[1]!.text, rows[2]!.text, rows[5]!.text], ["first", "second", "third"]);
+ assert.equal((await items(raw, "other-session"))[0]!.call_id, "repeated-call");
+ });
+});
diff --git a/test/postgres-items-operator.test.ts b/test/postgres-items-operator.test.ts
new file mode 100644
index 000000000..8282e9a69
--- /dev/null
+++ b/test/postgres-items-operator.test.ts
@@ -0,0 +1,203 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import pg from "pg";
+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";
+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";
+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";
+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
+
+const url = process.env.DATABASE_URL;
+const insertSession = `INSERT INTO sessions(id,type,scope_id,thread_ref,created_at) VALUES($1,'dm','personal:test',$1,1)`;
+const insertEntry = `INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at) VALUES($1,$2,'user',$3,'personal:test',1)`;
+
+test(
+ "items operator resumes bounded copies and reconciles updates, low keys, delayed commits and cascade deletes",
+ { skip: !url, timeout: 45000 },
+ async () => {
+ const pool = new pg.Pool({ connectionString: url });
+ const client = new pg.Client({ connectionString: url });
+ const writer = new pg.Client({ connectionString: url });
+ await client.connect();
+ await writer.connect();
+ try {
+ createPostgresSessionStore(url!);
+ createPostgresDirectoryStore(url!);
+ await applyPgMigrations(
+ pool,
+ registeredPgMigrations(url!).filter((m) => m.id !== "sessions/store/0035-items"),
+ );
+ await client.query(
+ "ALTER TABLE session_entries ADD CONSTRAINT operator_source_fk FOREIGN KEY(session_id) REFERENCES sessions(id) ON DELETE CASCADE",
+ );
+ await client.query(insertSession, ["z"]);
+ for (let i = 0; i < 8; i++) await client.query(insertEntry, ["z", i, JSON.stringify({ text: `line ${i}` })]);
+ await client.query("UPDATE session_entries SET type='soul' WHERE seq>=6");
+ const op = new ItemsCopyOperator(client, 2);
+ await op.prepare();
+ await assert.rejects(op.prepare(), /already exists/);
+ assert.equal(await op.copyBatch(), 2);
+ await writer.query("UPDATE session_entries SET payload='broken' WHERE session_id='z' AND seq=2");
+ const checkpoint = await op.status();
+ await assert.rejects(op.copyBatch(), /invalid input syntax/);
+ assert.equal((await op.status()).scanned, checkpoint.scanned);
+ await writer.query("UPDATE session_entries SET payload='{\"text\":\"fixed\"}' WHERE session_id='z' AND seq=2");
+ await writer.query(insertSession, ["a"]);
+ await writer.query(insertEntry, ["a", 0, '{"text":"lower key"}']);
+ await writer.query("UPDATE session_entries SET payload='{\"text\":\"edit\"}' WHERE session_id='z' AND seq=0");
+ await writer.query("DELETE FROM session_entries WHERE session_id='z' AND seq=1");
+ await writer.query("UPDATE session_entries SET seq=12 WHERE session_id='z' AND seq=4");
+ await assert.rejects(writer.query("TRUNCATE session_entries"), /TRUNCATE is disabled/);
+ const resumed = new ItemsCopyOperator(client, 2);
+ while (await resumed.copyBatch()) continue;
+ assert.equal(await resumed.completeCopy(), true);
+ assert.equal((await resumed.status()).phase, "catchup");
+ await writer.query("BEGIN");
+ await writer.query(insertSession, ["late"]);
+ await writer.query(insertEntry, ["late", 0, '{"text":"late commit"}']);
+ await client.query("UPDATE session_entries SET payload='{\"text\":\"later id\"}' WHERE session_id='z' AND seq=5");
+ while (await resumed.catchupBatch()) continue;
+ await writer.query("COMMIT");
+ while (await resumed.catchupBatch()) continue;
+ assert.equal((await resumed.status()).queued, "0");
+ assert.deepEqual(
+ (await client.query("SELECT session_id,seq::int,text FROM items ORDER BY session_id,seq")).rows,
+ [
+ { session_id: "a", seq: 0, text: "lower key" },
+ { session_id: "late", seq: 0, text: "late commit" },
+ { session_id: "z", seq: 0, text: "edit" },
+ { session_id: "z", seq: 2, text: "fixed" },
+ { session_id: "z", seq: 3, text: "line 3" },
+ { session_id: "z", seq: 5, text: "later id" },
+ { session_id: "z", seq: 12, text: "line 4" },
+ ],
+ );
+ await resumed.verify();
+ await client.query("UPDATE items SET text='corrupt' WHERE session_id='z' AND seq=0");
+ await assert.rejects(resumed.verify(), /verification found differences/);
+ await assert.rejects(resumed.finish(true), /phase changed/);
+ await writer.query("UPDATE session_entries SET payload=payload WHERE session_id='z' AND seq=0");
+ while (await resumed.catchupBatch()) continue;
+ await resumed.verify();
+ await writer.query("DELETE FROM sessions WHERE id='a'");
+ while (await resumed.catchupBatch()) continue;
+ assert.equal((await client.query("SELECT count(*)::int AS n FROM items WHERE session_id='a'")).rows[0].n, 0);
+ await resumed.verify();
+ await client.query("INSERT INTO participants(session_id,principal_id,valid_from) VALUES('z','ann',0)");
+ while (await resumed.catchupBatch()) continue;
+ assert.equal(
+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,
+ "ann",
+ );
+ await client.query("INSERT INTO participants(session_id,principal_id,valid_from) VALUES('z','bob',0)");
+ await client.query(
+ 'UPDATE session_entries SET payload=\'{"text":"edit","name":"Ann","ts":"1.1"}\' WHERE session_id=\'z\' AND seq=0',
+ );
+ await client.query(
+ "INSERT INTO directory_members(org_id,principal_id,display_name,display_name_lc,type) VALUES('o','ann','Ann','ann','internal')",
+ );
+ while (await resumed.catchupBatch()) continue;
+ assert.equal(
+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,
+ "ann",
+ );
+ await client.query("DELETE FROM directory_members WHERE principal_id='ann'");
+ while (await resumed.catchupBatch()) continue;
+ assert.equal(
+ (await client.query("SELECT author FROM items WHERE session_id='z' AND seq=0")).rows[0].author,
+ null,
+ );
+ await client.query(insertEntry, ["z", 15, '{"thinking":"thought"}']);
+ await client.query("UPDATE session_entries SET type='thinking' WHERE session_id='z' AND seq=15");
+ await client.query(
+ 'INSERT INTO legacy_tape_steps(session_id,seq,step) VALUES(\'z\',15,\'{"index":3,"model":"m"}\')',
+ );
+ while (await resumed.catchupBatch()) continue;
+ assert.equal((await client.query("SELECT step FROM items WHERE session_id='z' AND seq=15")).rows[0].step, 3);
+ await client.query(insertEntry, ["z", 20, '{"kind":"message_revision","ts":"9.1"}']);
+ await client.query("UPDATE session_entries SET type='system' WHERE session_id='z' AND seq=20");
+ while (await resumed.catchupBatch()) continue;
+ assert.equal(
+ (await client.query("SELECT target_seq FROM items WHERE session_id='z' AND seq=20")).rows[0].target_seq,
+ null,
+ );
+ await client.query(
+ 'UPDATE session_entries SET payload=\'{"text":"new first","ts":"9.1"}\' WHERE session_id=\'z\' AND seq=0',
+ );
+ await resumed.catchupBatch();
+ await client.query(
+ 'UPDATE session_entries SET payload=\'{"text":"newer first","ts":"9.1"}\' WHERE session_id=\'z\' AND seq=0',
+ );
+ while (await resumed.catchupBatch()) continue;
+ assert.equal(
+ (await client.query("SELECT target_seq::int FROM items WHERE session_id='z' AND seq=20")).rows[0].target_seq,
+ 0,
+ );
+ assert.equal(
+ (await client.query("SELECT text FROM items WHERE session_id='z' AND seq=0")).rows[0].text,
+ "newer first",
+ );
+ await resumed.verify();
+ await client.query("UPDATE session_entries SET payload=payload WHERE session_id='late'");
+ await writer.query("BEGIN");
+ await writer.query("DELETE FROM sessions WHERE id='late'");
+ await assert.rejects(resumed.catchupBatch(), (error) => (error as { code?: string }).code === "55P03");
+ await writer.query("COMMIT");
+ while (await resumed.catchupBatch()) continue;
+ await resumed.verify();
+ await assert.rejects(resumed.cleanup(), /cutover migration/);
+ await assert.rejects(resumed.finish(false), /writers-stopped/);
+ await client.query(
+ "INSERT INTO session_leases(session_id,token,expires_at) VALUES('z','busy',9223372036854775807)",
+ );
+ await assert.rejects(resumed.finish(true), /active session leases/);
+ await client.query("DELETE FROM session_leases WHERE session_id='z'");
+ await client.query("ALTER TABLE session_entries DISABLE TRIGGER items_copy_capture");
+ await assert.rejects(resumed.catchupBatch(), /capture is missing or disabled/);
+ await client.query("ALTER TABLE session_entries ENABLE TRIGGER items_copy_capture");
+ await resumed.verify();
+ await assert.rejects(resumed.finish(true), /build the items indexes/);
+ await assert.rejects(
+ client.query("CREATE UNIQUE INDEX CONCURRENTLY items_one_call_per_kind ON items(tenant_id)"),
+ /could not create unique index/,
+ );
+ await writer.query("SELECT pg_advisory_lock(hashtext('qm:schema-migrations'))");
+ try {
+ await resumed.indexes();
+ await resumed.indexes();
+ } finally {
+ await writer.query("SELECT pg_advisory_unlock(hashtext('qm:schema-migrations'))");
+ }
+ assert.match(
+ (await client.query("SELECT pg_get_indexdef('items_one_call_per_kind'::regclass) AS def")).rows[0].def,
+ /call_id\) WHERE \(call_id IS NOT NULL\)/,
+ );
+ assert.equal(
+ (
+ await client.query(
+ "SELECT count(*)::int AS n FROM pg_index WHERE indrelid='items'::regclass AND NOT indisvalid",
+ )
+ ).rows[0].n,
+ 0,
+ );
+ await resumed.finish(true);
+ assert.equal((await resumed.status()).phase, "ready");
+ assert.equal(
+ (await client.query("SELECT count(*)::int AS n FROM pg_indexes WHERE tablename='items'")).rows[0].n,
+ 7,
+ );
+ await resumed.finish(true);
+ await applyPgMigrations(pool, registeredPgMigrations(url!));
+ await client.query(
+ "INSERT INTO qm_schema_migrations(id,checksum) VALUES('sessions/store/0036-items-cutover','test')",
+ );
+ await assert.rejects(resumed.catchupBatch(), /cutover is already applied/);
+ await resumed.cleanup();
+ assert.equal((await client.query("SELECT to_regnamespace('items_copy_ops') AS schema")).rows[0].schema, null);
+ assert.ok((await client.query("SELECT count(*)::int AS n FROM items")).rows[0].n > 0);
+ } finally {
+ await writer.end();
+ await client.end();
+ await pool.end();
+ }
+ },
+);
diff --git a/test/postgres-items-parallel.test.ts b/test/postgres-items-parallel.test.ts
new file mode 100644
index 000000000..ad37de648
--- /dev/null
+++ b/test/postgres-items-parallel.test.ts
@@ -0,0 +1,71 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import pg from "pg";
+import { ItemsCopyOperator } from "../scripts/ops/items-copy.ts";
+import { createPostgresSessionStore } from "../src/sessions/postgres-session-store.ts";
+import { createPostgresDirectoryStore } from "../src/directory/postgres-directory-store.ts";
+import { applyPgMigrations, registeredPgMigrations } from "../src/persistence/pg-pool.ts";
+
+const url = process.env.DATABASE_URL;
+
+test(
+ "sliced copy workers resume one cursor, copy every row once and wait out schema migrations",
+ { skip: !url, timeout: 60000 },
+ async () => {
+ const pool = new pg.Pool({ connectionString: url });
+ const clients = await Promise.all(
+ Array.from({ length: 6 }, async () => {
+ const c = new pg.Client({ connectionString: url });
+ await c.connect();
+ return c;
+ }),
+ );
+ const [main, migrator, ...workers] = clients as [pg.Client, pg.Client, ...pg.Client[]];
+ try {
+ createPostgresSessionStore(url!);
+ createPostgresDirectoryStore(url!);
+ await applyPgMigrations(
+ pool,
+ registeredPgMigrations(url!).filter((m) => m.id !== "sessions/store/0035-items"),
+ );
+ await main.query(`INSERT INTO sessions(id,type,scope_id,thread_ref,created_at)
+ SELECT 's'||lpad(i::text,3,'0'),'dm','personal:test','s'||i,1 FROM generate_series(0,199) i`);
+ await main.query(`INSERT INTO session_entries(session_id,seq,type,payload,scope_label,created_at)
+ SELECT 's'||lpad((i%200)::text,3,'0'),i/200,'user',json_build_object('text','line '||i)::text,'personal:test',1
+ FROM generate_series(0,1999) i`);
+ const op = new ItemsCopyOperator(main, 50);
+ await op.prepare();
+ assert.equal(await op.copyBatch(), 50);
+ assert.equal(await op.split(4), 4);
+ assert.equal(await op.split(4), 4);
+ await assert.rejects(op.split(3), /already split into 4 slices/);
+ const slices = (await main.query("SELECT slice FROM items_copy_ops.slices ORDER BY slice")).rows.map(
+ (r) => r.slice,
+ );
+ await migrator.query("SELECT pg_advisory_lock(hashtext('qm:schema-migrations'))");
+ const release = setTimeout(
+ () => void migrator.query("SELECT pg_advisory_unlock(hashtext('qm:schema-migrations'))"),
+ 1500,
+ );
+ await Promise.all(
+ slices.map(async (slice, i) => {
+ const w = new ItemsCopyOperator(workers[i]!, 50);
+ while (await w.copyBatch(slice)) continue;
+ }),
+ );
+ clearTimeout(release);
+ assert.equal(await op.completeCopy(), true);
+ const status = await op.status();
+ assert.deepEqual(
+ [status.phase, Number(status.scanned), Number(status.initial_copied), status.done],
+ ["catchup", 2000, 2000, 4],
+ );
+ assert.equal((await main.query("SELECT count(DISTINCT (session_id,seq))::int AS n FROM items")).rows[0].n, 2000);
+ await op.verify();
+ assert.equal((await op.status()).phase, "verified");
+ } finally {
+ await Promise.all(clients.map((c) => c.end()));
+ await pool.end();
+ }
+ },
+);