Multiplayer agent harness for work.
Serialize migration startup and terminate completed migration tasks (#1193)
Regan Bell03918fc244bb
diff --git a/package.json b/package.jsonindex 6e5103398..c116a59d6 100644--- a/package.json+++ b/package.json@@ -36,7 +36,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.tsindex ed0a167e3..808bb2aa0 100644--- a/src/persistence/pg-pool.ts+++ b/src/persistence/pg-pool.ts@@ -1,6 +1,7 @@import { createHash } from "node:crypto";import { readFileSync } from "node:fs";import type { Pool, PoolClient } from "pg";+import { createKeyedQueue } from "../util/async.ts";import { errMessage, swallowAs } from "../util/errors.ts";export type { Pool, PoolClient };@@ -30,6 +31,8 @@ export function configurePgPooling(config: PgPoolingConfig): void {poolingConfig = { ...config };}+const migrationQueue = createKeyedQueue();+const sharedPools = new Map<string, { pool: Pool; users: number }>();async function retainPool(connectionString: string, kind: "query" | "session" | "migration"): Promise<Pool> {@@ -334,18 +337,20 @@ export async function migrateRegisteredPgSchemas(connectionString?: string): Pro: [...registeredMigrations.entries()];for (const [databaseUrl, registered] of databases) {if (!registered?.size) continue;- const pg = (await import("pg")).default;- const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });- pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));- try {- await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);- await applyPgMigrations(- pool,- [...registered.values()].sort((a, b) => a.id.localeCompare(b.id)),- );- } finally {- await pool.end();- }+ await migrationQueue(databaseUrl, async () => {+ const pg = (await import("pg")).default;+ const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });+ pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));+ try {+ await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);+ await applyPgMigrations(+ pool,+ [...registered.values()].sort((a, b) => a.id.localeCompare(b.id)),+ );+ } finally {+ await pool.end();+ }+ });}}@@ -394,12 +399,14 @@ export function createPgPool(let closed = false;const queryUrl = pooledDatabaseUrl(connectionString);async function withMigrationPool<T>(fn: (pool: Pool) => Promise<T>): Promise<T> {- const instance = await retainPool(connectionString, "migration");- try {- return await fn(instance);- } finally {- await releasePool(connectionString, instance, "migration");- }+ return migrationQueue(connectionString, async () => {+ const instance = await retainPool(connectionString, "migration");+ try {+ return await fn(instance);+ } finally {+ await releasePool(connectionString, instance, "migration");+ }+ });}async function ready(): Promise<void> {if (closed) throw new Error("Postgres store is closed");diff --git a/test/postgres-migration-startup.test.ts b/test/postgres-migration-startup.test.tsnew file mode 100644index 000000000..9b6757f39--- /dev/null+++ b/test/postgres-migration-startup.test.ts@@ -0,0 +1,83 @@+import assert from "node:assert/strict";+import { execFile } from "node:child_process";+import { randomUUID } from "node:crypto";+import { promisify } from "node:util";+import test from "node:test";+import pg from "pg";+import { createPgPool, migrateRegisteredPgSchemas, PG_MIGRATIONS_TABLE } from "../src/persistence/pg-pool.ts";++const databaseUrl = process.env.DATABASE_URL;+const skip = !databaseUrl;+const exec = promisify(execFile);++async function withDatabase(run: (url: string, admin: pg.Pool) => Promise<void>): Promise<void> {+ const parent = new pg.Pool({ connectionString: databaseUrl });+ const name = `migration_startup_${randomUUID().replaceAll("-", "")}`;+ await parent.query(`CREATE DATABASE ${name}`);+ const url = new URL(databaseUrl!);+ url.pathname = `/${name}`;+ const admin = new pg.Pool({ connectionString: url.toString() });+ try {+ await run(url.toString(), admin);+ } finally {+ await admin.end();+ await parent.query(`DROP DATABASE ${name} WITH (FORCE)`);+ await parent.end();+ }+}++test("slow lazy migrations queue with dynamic and bulk migrations before pool acquisition", { skip }, async () => {+ await withDatabase(async (url, admin) => {+ const slow = createPgPool(url, "startup/0001", ["SELECT pg_sleep(11)"]);+ const next = createPgPool(url, "startup/0002", ["CREATE TABLE startup_result(value INT)"]);+ try {+ const first = slow.q("SELECT 1");+ const second = next.q("SELECT 2");+ const dynamic = next.migrate({ id: "startup/0003", statements: ["INSERT INTO startup_result VALUES (3)"] });+ await Promise.all([first, second, dynamic, migrateRegisteredPgSchemas(url)]);+ assert.deepEqual((await admin.query("SELECT value FROM startup_result")).rows, [{ value: 3 }]);+ assert.equal((await admin.query(`SELECT count(*)::int AS count FROM ${PG_MIGRATIONS_TABLE}`)).rows[0].count, 3);+ await assert.rejects(next.migrate({ id: "startup/0004", statements: ["SELECT * FROM missing_table"] }));+ await next.migrate({ id: "startup/0005", statements: ["INSERT INTO startup_result VALUES (5)"] });+ assert.deepEqual((await admin.query("SELECT value FROM startup_result ORDER BY value")).rows, [+ { value: 3 },+ { value: 5 },+ ]);+ } finally {+ await Promise.all([slow.close(), next.close()]);+ }+ });+});++test("migration entrypoint exits after cold and repeat startup and rejects checksum drift", { skip }, async () => {+ await withDatabase(async (url, admin) => {+ const run = () =>+ exec(process.execPath, ["src/migrate-main.ts"], {+ cwd: new URL("..", import.meta.url),+ timeout: 60_000,+ env: {+ PATH: process.env.PATH,+ NODE_ENV: "test",+ DATABASE_URL: url,+ SANDBOX_BACKEND: "local",+ SESSION_STORE: "postgres",+ ORG_ID: "migration-test",+ CONNECTOR_SECRET_KEY: "test-connector-secret-0123456789abcdef",+ },+ });+ for (let attempt = 0; attempt < 2; attempt++) {+ const result = await run();+ assert.match(result.stdout, /\[qm:migrate\] database migrations applied/);+ }+ const applied = await admin.query(`SELECT id FROM ${PG_MIGRATIONS_TABLE} ORDER BY id`);+ assert.ok(applied.rowCount! > 10);+ await admin.query(`UPDATE ${PG_MIGRATIONS_TABLE} SET checksum = 'invalid' WHERE id = $1`, [applied.rows[0].id]);+ await assert.rejects(run(), (error: unknown) => {+ const failure = error as Error & { code: number; stdout: string; stderr: string };+ assert.equal(failure.code, 1);+ assert.match(failure.stderr, /checksum mismatch/);+ assert.doesNotMatch(failure.stdout, /\[qm:migrate\] database migrations applied/);+ return true;+ });+ });+});