Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Serialize migration startup and terminate completed migration tasks (#1193)

Regan Bell03918fc244bb
diff --git a/package.json b/package.json
index 6e5103398..c116a59d6 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
+ "test:pg": "node --test --test-concurrency=1 test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.ts
index ed0a167e3..808bb2aa0 100644
--- a/src/persistence/pg-pool.ts
+++ b/src/persistence/pg-pool.ts
@@ -1,6 +1,7 @@
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import type { Pool, PoolClient } from "pg";
+import { createKeyedQueue } from "../util/async.ts";
import { errMessage, swallowAs } from "../util/errors.ts";
export type { Pool, PoolClient };
@@ -30,6 +31,8 @@ export function configurePgPooling(config: PgPoolingConfig): void {
poolingConfig = { ...config };
}
+const migrationQueue = createKeyedQueue();
+
const sharedPools = new Map<string, { pool: Pool; users: number }>();
async function retainPool(connectionString: string, kind: "query" | "session" | "migration"): Promise<Pool> {
@@ -334,18 +337,20 @@ export async function migrateRegisteredPgSchemas(connectionString?: string): Pro
: [...registeredMigrations.entries()];
for (const [databaseUrl, registered] of databases) {
if (!registered?.size) continue;
- const pg = (await import("pg")).default;
- const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });
- pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));
- try {
- await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);
- await applyPgMigrations(
- pool,
- [...registered.values()].sort((a, b) => a.id.localeCompare(b.id)),
- );
- } finally {
- await pool.end();
- }
+ await migrationQueue(databaseUrl, async () => {
+ const pg = (await import("pg")).default;
+ const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });
+ pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));
+ try {
+ await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);
+ await applyPgMigrations(
+ pool,
+ [...registered.values()].sort((a, b) => a.id.localeCompare(b.id)),
+ );
+ } finally {
+ await pool.end();
+ }
+ });
}
}
@@ -394,12 +399,14 @@ export function createPgPool(
let closed = false;
const queryUrl = pooledDatabaseUrl(connectionString);
async function withMigrationPool<T>(fn: (pool: Pool) => Promise<T>): Promise<T> {
- const instance = await retainPool(connectionString, "migration");
- try {
- return await fn(instance);
- } finally {
- await releasePool(connectionString, instance, "migration");
- }
+ return migrationQueue(connectionString, async () => {
+ const instance = await retainPool(connectionString, "migration");
+ try {
+ return await fn(instance);
+ } finally {
+ await releasePool(connectionString, instance, "migration");
+ }
+ });
}
async function ready(): Promise<void> {
if (closed) throw new Error("Postgres store is closed");
diff --git a/test/postgres-migration-startup.test.ts b/test/postgres-migration-startup.test.ts
new file mode 100644
index 000000000..9b6757f39
--- /dev/null
+++ b/test/postgres-migration-startup.test.ts
@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { promisify } from "node:util";
+import test from "node:test";
+import pg from "pg";
+import { createPgPool, migrateRegisteredPgSchemas, PG_MIGRATIONS_TABLE } from "../src/persistence/pg-pool.ts";
+
+const databaseUrl = process.env.DATABASE_URL;
+const skip = !databaseUrl;
+const exec = promisify(execFile);
+
+async function withDatabase(run: (url: string, admin: pg.Pool) => Promise<void>): Promise<void> {
+ const parent = new pg.Pool({ connectionString: databaseUrl });
+ const name = `migration_startup_${randomUUID().replaceAll("-", "")}`;
+ await parent.query(`CREATE DATABASE ${name}`);
+ const url = new URL(databaseUrl!);
+ url.pathname = `/${name}`;
+ const admin = new pg.Pool({ connectionString: url.toString() });
+ try {
+ await run(url.toString(), admin);
+ } finally {
+ await admin.end();
+ await parent.query(`DROP DATABASE ${name} WITH (FORCE)`);
+ await parent.end();
+ }
+}
+
+test("slow lazy migrations queue with dynamic and bulk migrations before pool acquisition", { skip }, async () => {
+ await withDatabase(async (url, admin) => {
+ const slow = createPgPool(url, "startup/0001", ["SELECT pg_sleep(11)"]);
+ const next = createPgPool(url, "startup/0002", ["CREATE TABLE startup_result(value INT)"]);
+ try {
+ const first = slow.q("SELECT 1");
+ const second = next.q("SELECT 2");
+ const dynamic = next.migrate({ id: "startup/0003", statements: ["INSERT INTO startup_result VALUES (3)"] });
+ await Promise.all([first, second, dynamic, migrateRegisteredPgSchemas(url)]);
+ assert.deepEqual((await admin.query("SELECT value FROM startup_result")).rows, [{ value: 3 }]);
+ assert.equal((await admin.query(`SELECT count(*)::int AS count FROM ${PG_MIGRATIONS_TABLE}`)).rows[0].count, 3);
+ await assert.rejects(next.migrate({ id: "startup/0004", statements: ["SELECT * FROM missing_table"] }));
+ await next.migrate({ id: "startup/0005", statements: ["INSERT INTO startup_result VALUES (5)"] });
+ assert.deepEqual((await admin.query("SELECT value FROM startup_result ORDER BY value")).rows, [
+ { value: 3 },
+ { value: 5 },
+ ]);
+ } finally {
+ await Promise.all([slow.close(), next.close()]);
+ }
+ });
+});
+
+test("migration entrypoint exits after cold and repeat startup and rejects checksum drift", { skip }, async () => {
+ await withDatabase(async (url, admin) => {
+ const run = () =>
+ exec(process.execPath, ["src/migrate-main.ts"], {
+ cwd: new URL("..", import.meta.url),
+ timeout: 60_000,
+ env: {
+ PATH: process.env.PATH,
+ NODE_ENV: "test",
+ DATABASE_URL: url,
+ SANDBOX_BACKEND: "local",
+ SESSION_STORE: "postgres",
+ ORG_ID: "migration-test",
+ CONNECTOR_SECRET_KEY: "test-connector-secret-0123456789abcdef",
+ },
+ });
+ for (let attempt = 0; attempt < 2; attempt++) {
+ const result = await run();
+ assert.match(result.stdout, /\[qm:migrate\] database migrations applied/);
+ }
+ const applied = await admin.query(`SELECT id FROM ${PG_MIGRATIONS_TABLE} ORDER BY id`);
+ assert.ok(applied.rowCount! > 10);
+ await admin.query(`UPDATE ${PG_MIGRATIONS_TABLE} SET checksum = 'invalid' WHERE id = $1`, [applied.rows[0].id]);
+ await assert.rejects(run(), (error: unknown) => {
+ const failure = error as Error & { code: number; stdout: string; stderr: string };
+ assert.equal(failure.code, 1);
+ assert.match(failure.stderr, /checksum mismatch/);
+ assert.doesNotMatch(failure.stdout, /\[qm:migrate\] database migrations applied/);
+ return true;
+ });
+ });
+});