Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Run Postgres tests in parallel, one fresh database per test file (#1780)

Regan Belle93b725d0e31
diff --git a/package.json b/package.json
index 6b1f85b92..2ef1f4de1 100644
--- a/package.json
+++ b/package.json
@@ -46,7 +46,7 @@
"sandbox:local:build": "bash scripts/local-sandbox-build.sh",
"sandbox:porter:build": "docker build --platform linux/amd64 -t \"${PORTER_SANDBOX_IMAGE:?Set PORTER_SANDBOX_IMAGE to the registry tag to publish}\" porter && docker push \"$PORTER_SANDBOX_IMAGE\"",
"deploy:porter-runner-image": "docker build --platform linux/amd64 -t \"${PORTER_DEPLOY_RUNNER_IMAGE:?Set PORTER_DEPLOY_RUNNER_IMAGE to the registry tag to publish}\" -f deploy/porter-runner/Dockerfile . && docker push \"$PORTER_DEPLOY_RUNNER_IMAGE\"",
- "test:pg": "node --test --test-concurrency=1 test/readiness.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-migration-startup.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/postgres-instance-registry.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
+ "test:pg": "node scripts/run-pg-tests.mjs test/cron-queue.test.ts test/postgres-migration-startup.test.ts test/postgres-swarms.test.ts test/postgres-store.test.ts test/postgres-instance-registry.test.ts test/readiness.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-file-upload-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/postgres-background-ownership.test.ts test/postgres-resource-search.test.ts test/deploy-store-pg.test.ts test/postgres-cron-fires.test.ts test/postgres-migration-adoption.test.ts test/postgres-migrations.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-broker-sessions.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-listener.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/loop-ingress-pg.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/slack-mirror-fidelity.test.ts test/run-retry-delay.test.ts test/postgres-run-stream.test.ts test/migrate-main.test.ts",
"livetest": "node scripts/api-livetest.ts",
"bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts",
"prepare": "node .husky/install.mjs",
diff --git a/scripts/run-pg-tests.mjs b/scripts/run-pg-tests.mjs
new file mode 100644
index 000000000..c0ff14f0f
--- /dev/null
+++ b/scripts/run-pg-tests.mjs
@@ -0,0 +1,146 @@
+#!/usr/bin/env node
+import { spawn } from "node:child_process";
+import { randomBytes } from "node:crypto";
+import { availableParallelism } from "node:os";
+import process from "node:process";
+import { parseArgs } from "node:util";
+import pg from "pg";
+
+const { values, positionals: files } = parseArgs({
+ allowPositionals: true,
+ options: {
+ jobs: { type: "string", default: process.env.PG_TEST_JOBS ?? String(Math.min(4, availableParallelism())) },
+ timeout: { type: "string", default: "600" },
+ },
+});
+
+const jobs = Number(values.jobs);
+const timeoutSeconds = Number(values.timeout);
+if (!Number.isInteger(jobs) || jobs < 1) throw new Error(`--jobs must be a positive integer, got ${values.jobs}`);
+if (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) {
+ throw new Error(`--timeout must be a positive number of seconds, got ${values.timeout}`);
+}
+if (files.length === 0) throw new Error("Pass the Postgres test files to run");
+
+const baseUrl = process.env.DATABASE_URL;
+const runId = `qmt_${process.pid}_${randomBytes(4).toString("hex")}`;
+const children = new Set();
+const admin = baseUrl ? new pg.Pool({ connectionString: baseUrl, max: jobs }) : null;
+let stopping = false;
+let ended = null;
+
+function endPool() {
+ ended ??= admin?.end();
+ return ended;
+}
+
+function killGroup(child) {
+ if (!child.pid) return;
+ try {
+ process.kill(-child.pid, "SIGKILL");
+ } catch {
+ child.kill("SIGKILL");
+ }
+}
+
+function databaseUrl(name) {
+ const url = new URL(baseUrl);
+ url.pathname = `/${name}`;
+ return url.toString();
+}
+
+async function dropDatabase(name) {
+ const { rows } = await admin.query(
+ "SELECT datname FROM pg_database WHERE datname = $1 OR starts_with(datname, $1 || '_') ORDER BY datname DESC",
+ [name],
+ );
+ for (const { datname } of rows) {
+ await admin.query(`DROP DATABASE IF EXISTS "${datname.replaceAll('"', '""')}" WITH (FORCE)`);
+ }
+}
+
+function runFile(file, env) {
+ return new Promise((resolve) => {
+ const started = Date.now();
+ let output = "";
+ let settled = false;
+ const finish = (ok) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ children.delete(child);
+ resolve({ file, ok, ms: Date.now() - started, output });
+ };
+ const child = spawn(process.execPath, ["--test", file], { detached: true, env: { ...process.env, ...env } });
+ children.add(child);
+ child.stdout?.on("data", (chunk) => (output += chunk));
+ child.stderr?.on("data", (chunk) => (output += chunk));
+ const timer = setTimeout(() => {
+ output += `\n${file} exceeded ${timeoutSeconds}s and was killed\n`;
+ killGroup(child);
+ }, timeoutSeconds * 1000);
+ child.on("error", (error) => {
+ output += `\n${file} could not run: ${error.message || error.code || error}\n`;
+ finish(false);
+ });
+ child.on("exit", () => killGroup(child));
+ child.on("close", (code, signal) => finish(code === 0 && !signal));
+ });
+}
+
+async function runIsolated(file, index) {
+ if (!admin) return runFile(file, {});
+ const name = `${runId}_${index}`;
+ try {
+ await admin.query(`CREATE DATABASE ${name}`);
+ if (stopping) return { file, ok: false, ms: 0, output: "" };
+ return await runFile(file, { DATABASE_URL: databaseUrl(name) });
+ } finally {
+ await dropDatabase(name);
+ }
+}
+
+const started = Date.now();
+const results = [];
+let next = 0;
+const work = Promise.all(
+ Array.from({ length: Math.min(jobs, files.length) }, async () => {
+ while (!stopping && next < files.length) {
+ const index = next++;
+ const result = await runIsolated(files[index], index).catch((error) => ({
+ file: files[index],
+ ok: false,
+ ms: 0,
+ output: `\n${files[index]} could not run: ${error.message || error.code || error}\n`,
+ }));
+ if (stopping) return;
+ results.push(result);
+ process.stdout.write(`\n# ${result.ok ? "ok" : "FAIL"} ${result.file} (${(result.ms / 1000).toFixed(1)}s)\n`);
+ process.stdout.write(result.output);
+ }
+ }),
+);
+
+for (const signal of ["SIGINT", "SIGTERM"]) {
+ process.on(signal, () => {
+ if (stopping) return;
+ stopping = true;
+ for (const child of children) killGroup(child);
+ void work.then(async () => {
+ await endPool();
+ process.exit(signal === "SIGINT" ? 130 : 143);
+ });
+ });
+}
+
+await work;
+if (stopping) await new Promise(() => {});
+await endPool();
+
+const failed = results.filter((result) => !result.ok);
+const summedMs = results.reduce((sum, result) => sum + result.ms, 0);
+process.stdout.write(
+ `\n# ${results.length - failed.length}/${results.length} files passed with ${jobs} jobs in ${((Date.now() - started) / 1000).toFixed(1)}s (${(summedMs / 1000).toFixed(1)}s summed)\n`,
+);
+for (const result of failed) process.stdout.write(`# FAIL ${result.file}\n`);
+process.exitCode = failed.length > 0 ? 1 : 0;
diff --git a/test/leader-lease.test.ts b/test/leader-lease.test.ts
index d4c699458..c932917ce 100644
--- a/test/leader-lease.test.ts
+++ b/test/leader-lease.test.ts
@@ -150,6 +150,7 @@ test(
SELECT pg_terminate_backend(l.pid) AS ok
FROM pg_locks l, k
WHERE l.locktype = 'advisory' AND l.granted
+ AND l.database = (SELECT oid FROM pg_database WHERE datname = current_database())
AND l.classid::bigint = ((k.v >> 32) & 4294967295) AND l.objid::bigint = (k.v & 4294967295)`,
);
assert.equal(killed.length, 1, "found and terminated the holder's lock connection");
diff --git a/test/migrate-principals.test.ts b/test/migrate-principals.test.ts
index 366378cb4..8a3bebe4f 100644
--- a/test/migrate-principals.test.ts
+++ b/test/migrate-principals.test.ts
@@ -2,6 +2,7 @@ import { test, before, after } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";
import { join } from "node:path";
+import { randomBytes } from "node:crypto";
import { tmpdir } from "node:os";
import type { Pool } from "pg";
import { mergeDirs, runMigration } from "../scripts/migrate-principals-to-email.mjs";
@@ -12,6 +13,8 @@ const skip = BASE_URL ? false : "set DATABASE_URL (a Postgres) to run the migrat
const MAPPING = { U9MIGA: "[email protected]", U9MIGB: "[email protected]" };
const silent = () => {};
+const parentDatabase = BASE_URL ? new URL(BASE_URL).pathname.slice(1) : "";
+const database = `${/^qmt_\w+$/.test(parentDatabase) ? parentDatabase : "qm"}_migtest_${randomBytes(4).toString("hex")}`;
let testUrl = "";
let pool: Pool | null = null;
@@ -19,11 +22,10 @@ before(async () => {
if (!BASE_URL) return;
const pg = (await import("pg")).default;
const admin = new pg.Pool({ connectionString: BASE_URL });
- await admin.query("DROP DATABASE IF EXISTS qm_migtest WITH (FORCE)");
- await admin.query("CREATE DATABASE qm_migtest");
+ await admin.query(`CREATE DATABASE "${database}"`);
await admin.end();
const url = new URL(BASE_URL);
- url.pathname = "/qm_migtest";
+ url.pathname = `/${database}`;
testUrl = url.toString();
pool = new pg.Pool({ connectionString: testUrl });
await pool.query(`
@@ -34,6 +36,11 @@ before(async () => {
after(async () => {
await pool?.end();
+ if (!BASE_URL) return;
+ const pg = (await import("pg")).default;
+ const admin = new pg.Pool({ connectionString: BASE_URL });
+ await admin.query(`DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`);
+ await admin.end();
});
async function reset(): Promise<void> {