Multiplayer agent harness for work.
Run Postgres tests in parallel, one fresh database per test file (#1780)
Regan Belle93b725d0e31
diff --git a/package.json b/package.jsonindex 6b1f85b92..2ef1f4de1 100644--- a/package.json+++ b/package.json@@ -46,7 +46,7 @@"sandbox:local:build": "bash scripts/local-sandbox-build.sh","sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"","deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",- "test:pg": "node --test --test-concurrency=1 test/readiness.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",+ "test:pg": "node scripts/run-pg-tests.mjs test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts","livetest": "node scripts/api-livetest.ts","bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts","prepare": "node .husky/install.mjs",diff --git a/scripts/run-pg-tests.mjs b/scripts/run-pg-tests.mjsnew file mode 100644index 000000000..c0ff14f0f--- /dev/null+++ b/scripts/run-pg-tests.mjs@@ -0,0 +1,146 @@+#!/usr/bin/env node+import { spawn } from "node:child_process";+import { randomBytes } from "node:crypto";+import { availableParallelism } from "node:os";+import process from "node:process";+import { parseArgs } from "node:util";+import pg from "pg";++const { values, positionals: files } = parseArgs({+ allowPositionals: true,+ options: {+ jobs: { type: "string", default: process.env.PG_TEST_JOBS ?? String(Math.min(4, availableParallelism())) },+ timeout: { type: "string", default: "600" },+ },+});++const jobs = Number(values.jobs);+const timeoutSeconds = Number(values.timeout);+if (!Number.isInteger(jobs) || jobs < 1) throw new Error(`--jobs must be a positive integer, got ${values.jobs}`);+if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) {+ throw new Error(`--timeout must be a positive number of seconds, got ${values.timeout}`);+}+if (files.length === 0) throw new Error("Pass the Postgres test files to run");++const baseUrl = process.env.DATABASE_URL;+const runId = `qmt_${process.pid}_${randomBytes(4).toString("hex")}`;+const children = new Set();+const admin = baseUrl ? new pg.Pool({ connectionString: baseUrl, max: jobs }) : null;+let stopping = false;+let ended = null;++function endPool() {+ ended ??= admin?.end();+ return ended;+}++function killGroup(child) {+ if (!child.pid) return;+ try {+ process.kill(-child.pid, "SIGKILL");+ } catch {+ child.kill("SIGKILL");+ }+}++function databaseUrl(name) {+ const url = new URL(baseUrl);+ url.pathname = `/${name}`;+ return url.toString();+}++async function dropDatabase(name) {+ const { rows } = await admin.query(+ "SELECT datname FROM pg_database WHERE datname = $1 OR starts_with(datname, $1 || '_') ORDER BY datname DESC",+ [name],+ );+ for (const { datname } of rows) {+ await admin.query(`DROP DATABASE IF EXISTS "${datname.replaceAll('"', '""')}" WITH (FORCE)`);+ }+}++function runFile(file, env) {+ return new Promise((resolve) => {+ const started = Date.now();+ let output = "";+ let settled = false;+ const finish = (ok) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ children.delete(child);+ resolve({ file, ok, ms: Date.now() - started, output });+ };+ const child = spawn(process.execPath, ["--test", file], { detached: true, env: { ...process.env, ...env } });+ children.add(child);+ child.stdout?.on("data", (chunk) => (output += chunk));+ child.stderr?.on("data", (chunk) => (output += chunk));+ const timer = setTimeout(() => {+ output += `\n${file} exceeded ${timeoutSeconds}s and was killed\n`;+ killGroup(child);+ }, timeoutSeconds * 1000);+ child.on("error", (error) => {+ output += `\n${file} could not run: ${error.message || error.code || error}\n`;+ finish(false);+ });+ child.on("exit", () => killGroup(child));+ child.on("close", (code, signal) => finish(code === 0 && !signal));+ });+}++async function runIsolated(file, index) {+ if (!admin) return runFile(file, {});+ const name = `${runId}_${index}`;+ try {+ await admin.query(`CREATE DATABASE ${name}`);+ if (stopping) return { file, ok: false, ms: 0, output: "" };+ return await runFile(file, { DATABASE_URL: databaseUrl(name) });+ } finally {+ await dropDatabase(name);+ }+}++const started = Date.now();+const results = [];+let next = 0;+const work = Promise.all(+ Array.from({ length: Math.min(jobs, files.length) }, async () => {+ while (!stopping && next < files.length) {+ const index = next++;+ const result = await runIsolated(files[index], index).catch((error) => ({+ file: files[index],+ ok: false,+ ms: 0,+ output: `\n${files[index]} could not run: ${error.message || error.code || error}\n`,+ }));+ if (stopping) return;+ results.push(result);+ process.stdout.write(`\n# ${result.ok ? "ok" : "FAIL"} ${result.file} (${(result.ms / 1000).toFixed(1)}s)\n`);+ process.stdout.write(result.output);+ }+ }),+);++for (const signal of ["SIGINT", "SIGTERM"]) {+ process.on(signal, () => {+ if (stopping) return;+ stopping = true;+ for (const child of children) killGroup(child);+ void work.then(async () => {+ await endPool();+ process.exit(signal === "SIGINT" ? 130 : 143);+ });+ });+}++await work;+if (stopping) await new Promise(() => {});+await endPool();++const failed = results.filter((result) => !result.ok);+const summedMs = results.reduce((sum, result) => sum + result.ms, 0);+process.stdout.write(+ `\n# ${results.length - failed.length}/${results.length} files passed with ${jobs} jobs in ${((Date.now() - started) / 1000).toFixed(1)}s (${(summedMs / 1000).toFixed(1)}s summed)\n`,+);+for (const result of failed) process.stdout.write(`# FAIL ${result.file}\n`);+process.exitCode = failed.length > 0 ? 1 : 0;diff --git a/test/leader-lease.test.ts b/test/leader-lease.test.tsindex d4c699458..c932917ce 100644--- a/test/leader-lease.test.ts+++ b/test/leader-lease.test.ts@@ -150,6 +150,7 @@ test(SELECT pg_terminate_backend(l.pid) AS okFROM pg_locks l, kWHERE l.locktype = 'advisory' AND l.granted+ AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())AND l.classid::bigint = ((k.v >> 32) & 4294967295) AND l.objid::bigint = (k.v & 4294967295)`,);assert.equal(killed.length, 1, "found and terminated the holder's lock connection");diff --git a/test/migrate-principals.test.ts b/test/migrate-principals.test.tsindex 366378cb4..8a3bebe4f 100644--- a/test/migrate-principals.test.ts+++ b/test/migrate-principals.test.ts@@ -2,6 +2,7 @@ import { test, before, after } from "node:test";import assert from "node:assert/strict";import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";import { join } from "node:path";+import { randomBytes } from "node:crypto";import { tmpdir } from "node:os";import type { Pool } from "pg";import { mergeDirs, runMigration } from "../scripts/migrate-principals-to-email.mjs";@@ -12,6 +13,8 @@ const skip = BASE_URL ? false : "set DATABASE_URL (a Postgres) to run the migratconst MAPPING = { U9MIGA: "[email protected]", U9MIGB: "[email protected]" };const silent = () => {};+const parentDatabase = BASE_URL ? new URL(BASE_URL).pathname.slice(1) : "";+const database = `${/^qmt_\w+$/.test(parentDatabase) ? parentDatabase : "qm"}_migtest_${randomBytes(4).toString("hex")}`;let testUrl = "";let pool: Pool | null = null;@@ -19,11 +22,10 @@ before(async () => {if (!BASE_URL) return;const pg = (await import("pg")).default;const admin = new pg.Pool({ connectionString: BASE_URL });- await admin.query("DROP DATABASE IF EXISTS qm_migtest WITH (FORCE)");- await admin.query("CREATE DATABASE qm_migtest");+ await admin.query(`CREATE DATABASE "${database}"`);await admin.end();const url = new URL(BASE_URL);- url.pathname = "/qm_migtest";+ url.pathname = `/${database}`;testUrl = url.toString();pool = new pg.Pool({ connectionString: testUrl });await pool.query(`@@ -34,6 +36,11 @@ before(async () => {after(async () => {await pool?.end();+ if (!BASE_URL) return;+ const pg = (await import("pg")).default;+ const admin = new pg.Pool({ connectionString: BASE_URL });+ await admin.query(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`);+ await admin.end();});async function reset(): Promise<void> {