Multiplayer agent harness for work.
Survive database blips without crashing the core (#1231)
Joshua France573dd650716e
diff --git a/src/egress-authz-main.ts b/src/egress-authz-main.tsindex e484e80c9..6b882ea26 100644--- a/src/egress-authz-main.ts+++ b/src/egress-authz-main.ts@@ -8,6 +8,7 @@ import { createPostgresEgressAuditSink } from "./admin/postgres-egress-audit-sinimport { signedRequestHeaders } from "./auth/source-auth-sign.ts";import { createSweeper } from "./util/sweeper.ts";import { errMessage } from "./util/errors.ts";+import { shutdownOnUncaught } from "./util/process-guard.ts";import { numEnv } from "./config.ts";import type { EgressPolicy, ScopeId } from "./types.ts";import { isPrivateNetworkIp } from "./util/network.ts";@@ -245,11 +246,14 @@ function main(): void {const tokenless = process.env.EGRESS_TOKENLESS === "open" ? ("open" as const) : ("deny" as const);const server = buildEgressAuthzServer({ ...(capabilitySecret ? { capabilitySecret } : {}), audit, tokenless });server.listen(port, "127.0.0.1", () => console.log(`[egress-authz] listening on 127.0.0.1:${port}`));- for (const sig of ["SIGTERM", "SIGINT"] as const) {- process.on(sig, () =>- server.close(() => void (relay ? relay.flush() : Promise.resolve()).finally(() => process.exit(0))),- );- }+ let shuttingDown = false;+ const shutdown = (): void => {+ if (shuttingDown) return;+ shuttingDown = true;+ server.close(() => void (relay ? relay.flush() : Promise.resolve()).finally(() => process.exit()));+ };+ for (const sig of ["SIGTERM", "SIGINT"] as const) process.on(sig, shutdown);+ shutdownOnUncaught("egress-authz", shutdown);}if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) main();diff --git a/src/index.ts b/src/index.tsindex b39b49496..26e891ab5 100644--- a/src/index.ts+++ b/src/index.ts@@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto";import { lookup } from "node:dns/promises";import { loadConfig } from "./config.ts";import { buildApp, serverDeps, stopWithBackstop } from "./wiring.ts";+import { shutdownOnUncaught } from "./util/process-guard.ts";import { createServer } from "./api/server.ts";import { dockerDaemonFailure } from "./deploy/docker-deploy-provider.ts";import { errMessage } from "./util/errors.ts";@@ -138,3 +139,4 @@ function shutdown(signal: string): void {}process.on("SIGINT", () => shutdown("SIGINT"));process.on("SIGTERM", () => shutdown("SIGTERM"));+shutdownOnUncaught("qm", shutdown);diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.tsindex 808bb2aa0..e085e5c7f 100644--- a/src/persistence/pg-pool.ts+++ b/src/persistence/pg-pool.ts@@ -55,12 +55,17 @@ async function retainPool(connectionString: string, kind: "query" | "session" |url = parsed.toString();ssl = { ssl: { ca: poolingConfig.caCert } };}- const pool = new pg.Pool({ connectionString: url, ...ssl, max, connectionTimeoutMillis: 10_000 });- pool.on("error", (error) => console.error("[pg] idle client error:", errMessage(error)));+ const pool = guardedPool(new pg.Pool({ connectionString: url, ...ssl, max, connectionTimeoutMillis: 10_000 }));sharedPools.set(key, { pool, users: 1 });return pool;}+function guardedPool(pool: Pool): Pool {+ pool.on("error", () => {});+ pool.on("connect", (client) => client.on("error", (error) => console.error("[pg] client error:", errMessage(error))));+ return pool;+}+async function releasePool(connectionString: string,pool: Pool,@@ -339,8 +344,7 @@ export async function migrateRegisteredPgSchemas(connectionString?: string): Proif (!registered?.size) continue;await migrationQueue(databaseUrl, async () => {const pg = (await import("pg")).default;- const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });- pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));+ const pool = guardedPool(new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() }));try {await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);await applyPgMigrations(diff --git a/src/runs/worker-main.ts b/src/runs/worker-main.tsindex 0937cfa48..c6d9c2d11 100644--- a/src/runs/worker-main.ts+++ b/src/runs/worker-main.ts@@ -1,5 +1,6 @@import { loadConfig } from "../config.ts";import { buildApp, stopWithBackstop } from "../wiring.ts";+import { shutdownOnUncaught } from "../util/process-guard.ts";import { migrateRegisteredPgSchemas } from "../persistence/pg-pool.ts";const config = loadConfig();@@ -21,3 +22,4 @@ function shutdown(signal: string): void {}process.on("SIGINT", () => shutdown("SIGINT"));process.on("SIGTERM", () => shutdown("SIGTERM"));+shutdownOnUncaught("qm:worker", shutdown);diff --git a/src/util/process-guard.ts b/src/util/process-guard.tsnew file mode 100644index 000000000..378761350--- /dev/null+++ b/src/util/process-guard.ts@@ -0,0 +1,11 @@+import { errMessage } from "./errors.ts";++export function shutdownOnUncaught(label: string, shutdown: (reason: string) => void): void {+ const onFatal = (kind: string) => (e: unknown) => {+ console.error(`[${label}] ${kind}; draining and exiting:`, e instanceof Error && e.stack ? e.stack : errMessage(e));+ process.exitCode = 1;+ shutdown(kind);+ };+ process.on("uncaughtException", onFatal("uncaught exception"));+ process.on("unhandledRejection", onFatal("unhandled rejection"));+}diff --git a/src/wiring.ts b/src/wiring.tsindex f9139bfab..830fc82e9 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -386,14 +386,14 @@ export function stopWithBackstop(): void {const hardExit = setTimeout(() => {console.error(`[${label}] drain overran; releasing in-flight leases before forced exit`);- void Promise.race([runtime.releaseInFlightRuns(), sleep(3_000, { unref: true })]).finally(() => process.exit(0));+ void Promise.race([runtime.releaseInFlightRuns(), sleep(3_000, { unref: true })]).finally(() => process.exit());}, shutdownDrainMs + 5_000);hardExit.unref();void runtime.stop().then(() => {clearTimeout(hardExit);beforeExit?.();- process.exit(0);+ process.exit();},(e: unknown) => {console.error(`[${label}] graceful stop failed: ${errMessage(e)}`);