Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Survive database blips without crashing the core (#1231)

Joshua France573dd650716e
diff --git a/src/egress-authz-main.ts b/src/egress-authz-main.ts
index e484e80c9..6b882ea26 100644
--- a/src/egress-authz-main.ts
+++ b/src/egress-authz-main.ts
@@ -8,6 +8,7 @@ import { createPostgresEgressAuditSink } from "./admin/postgres-egress-audit-sin
import { signedRequestHeaders } from "./auth/source-auth-sign.ts";
import { createSweeper } from "./util/sweeper.ts";
import { errMessage } from "./util/errors.ts";
+import { shutdownOnUncaught } from "./util/process-guard.ts";
import { numEnv } from "./config.ts";
import type { EgressPolicy, ScopeId } from "./types.ts";
import { isPrivateNetworkIp } from "./util/network.ts";
@@ -245,11 +246,14 @@ function main(): void {
const tokenless = process.env.EGRESS_TOKENLESS === "open" ? ("open" as const) : ("deny" as const);
const server = buildEgressAuthzServer({ ...(capabilitySecret ? { capabilitySecret } : {}), audit, tokenless });
server.listen(port, "127.0.0.1", () => console.log(`[egress-authz] listening on 127.0.0.1:${port}`));
- for (const sig of ["SIGTERM", "SIGINT"] as const) {
- process.on(sig, () =>
- server.close(() => void (relay ? relay.flush() : Promise.resolve()).finally(() => process.exit(0))),
- );
- }
+ let shuttingDown = false;
+ const shutdown = (): void => {
+ if (shuttingDown) return;
+ shuttingDown = true;
+ server.close(() => void (relay ? relay.flush() : Promise.resolve()).finally(() => process.exit()));
+ };
+ for (const sig of ["SIGTERM", "SIGINT"] as const) process.on(sig, shutdown);
+ shutdownOnUncaught("egress-authz", shutdown);
}
if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) main();
diff --git a/src/index.ts b/src/index.ts
index b39b49496..26e891ab5 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -3,6 +3,7 @@ import { randomBytes } from "node:crypto";
import { lookup } from "node:dns/promises";
import { loadConfig } from "./config.ts";
import { buildApp, serverDeps, stopWithBackstop } from "./wiring.ts";
+import { shutdownOnUncaught } from "./util/process-guard.ts";
import { createServer } from "./api/server.ts";
import { dockerDaemonFailure } from "./deploy/docker-deploy-provider.ts";
import { errMessage } from "./util/errors.ts";
@@ -138,3 +139,4 @@ function shutdown(signal: string): void {
}
process.on("SIGINT", () => shutdown("SIGINT"));
process.on("SIGTERM", () => shutdown("SIGTERM"));
+shutdownOnUncaught("qm", shutdown);
diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.ts
index 808bb2aa0..e085e5c7f 100644
--- a/src/persistence/pg-pool.ts
+++ b/src/persistence/pg-pool.ts
@@ -55,12 +55,17 @@ async function retainPool(connectionString: string, kind: "query" | "session" |
url = parsed.toString();
ssl = { ssl: { ca: poolingConfig.caCert } };
}
- const pool = new pg.Pool({ connectionString: url, ...ssl, max, connectionTimeoutMillis: 10_000 });
- pool.on("error", (error) => console.error("[pg] idle client error:", errMessage(error)));
+ const pool = guardedPool(new pg.Pool({ connectionString: url, ...ssl, max, connectionTimeoutMillis: 10_000 }));
sharedPools.set(key, { pool, users: 1 });
return pool;
}
+function guardedPool(pool: Pool): Pool {
+ pool.on("error", () => {});
+ pool.on("connect", (client) => client.on("error", (error) => console.error("[pg] client error:", errMessage(error))));
+ return pool;
+}
+
async function releasePool(
connectionString: string,
pool: Pool,
@@ -339,8 +344,7 @@ export async function migrateRegisteredPgSchemas(connectionString?: string): Pro
if (!registered?.size) continue;
await migrationQueue(databaseUrl, async () => {
const pg = (await import("pg")).default;
- const pool = new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() });
- pool.on("error", (error) => console.error("[pg] migration pool error:", errMessage(error)));
+ const pool = guardedPool(new pg.Pool({ connectionString: databaseUrl, ...pgCaOptions() }));
try {
await applyPgMaintenance(pool, [...(registeredPreMigrationMaintenance.get(databaseUrl)?.values() ?? [])]);
await applyPgMigrations(
diff --git a/src/runs/worker-main.ts b/src/runs/worker-main.ts
index 0937cfa48..c6d9c2d11 100644
--- a/src/runs/worker-main.ts
+++ b/src/runs/worker-main.ts
@@ -1,5 +1,6 @@
import { loadConfig } from "../config.ts";
import { buildApp, stopWithBackstop } from "../wiring.ts";
+import { shutdownOnUncaught } from "../util/process-guard.ts";
import { migrateRegisteredPgSchemas } from "../persistence/pg-pool.ts";
const config = loadConfig();
@@ -21,3 +22,4 @@ function shutdown(signal: string): void {
}
process.on("SIGINT", () => shutdown("SIGINT"));
process.on("SIGTERM", () => shutdown("SIGTERM"));
+shutdownOnUncaught("qm:worker", shutdown);
diff --git a/src/util/process-guard.ts b/src/util/process-guard.ts
new file mode 100644
index 000000000..378761350
--- /dev/null
+++ b/src/util/process-guard.ts
@@ -0,0 +1,11 @@
+import { errMessage } from "./errors.ts";
+
+export function shutdownOnUncaught(label: string, shutdown: (reason: string) => void): void {
+ const onFatal = (kind: string) => (e: unknown) => {
+ console.error(`[${label}] ${kind}; draining and exiting:`, e instanceof Error && e.stack ? e.stack : errMessage(e));
+ process.exitCode = 1;
+ shutdown(kind);
+ };
+ process.on("uncaughtException", onFatal("uncaught exception"));
+ process.on("unhandledRejection", onFatal("unhandled rejection"));
+}
diff --git a/src/wiring.ts b/src/wiring.ts
index f9139bfab..830fc82e9 100644
--- a/src/wiring.ts
+++ b/src/wiring.ts
@@ -386,14 +386,14 @@ export function stopWithBackstop(
): void {
const hardExit = setTimeout(() => {
console.error(`[${label}] drain overran; releasing in-flight leases before forced exit`);
- void Promise.race([runtime.releaseInFlightRuns(), sleep(3_000, { unref: true })]).finally(() => process.exit(0));
+ void Promise.race([runtime.releaseInFlightRuns(), sleep(3_000, { unref: true })]).finally(() => process.exit());
}, shutdownDrainMs + 5_000);
hardExit.unref();
void runtime.stop().then(
() => {
clearTimeout(hardExit);
beforeExit?.();
- process.exit(0);
+ process.exit();
},
(e: unknown) => {
console.error(`[${label}] graceful stop failed: ${errMessage(e)}`);