The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(db): reap idle pool connections, name the pool, and end it on shutdown (#12956)
Jannes Stubbemann023e640a7ee3
diff --git a/doc/DATABASE.md b/doc/DATABASE.mdindex 340f6e7d27..5f1c0326d0 100644--- a/doc/DATABASE.md+++ b/doc/DATABASE.md@@ -122,8 +122,10 @@ All of these are optional; when unset, the driver defaults apply and behavior is```shDATABASE_PREPARED_STATEMENTS=false # required for transaction-mode poolers; default: enabledDATABASE_POOL_MAX=25 # connection pool size; default: 10-DATABASE_IDLE_TIMEOUT_SECONDS=60 # close idle pooled connections; default: keep open+DATABASE_IDLE_TIMEOUT_SECONDS=60 # close idle pooled connections; default: 60 (0 = keep open)DATABASE_CONNECT_TIMEOUT_SECONDS=10 # default: 30+DATABASE_MAX_LIFETIME_SECONDS=1800 # recycle a pooled connection after this long; default: 30-60 min (random)+DATABASE_APPLICATION_NAME=paperclip # application_name in pg_stat_activity; default: paperclip```### Push the schemadiff --git a/docs/deploy/database.md b/docs/deploy/database.mdindex 0d4ad5731e..5f25489d8e 100644--- a/docs/deploy/database.md+++ b/docs/deploy/database.md@@ -62,7 +62,22 @@ If using connection pooling (transaction mode), disable prepared statements viaDATABASE_PREPARED_STATEMENTS=false```-Related optional client tuning (driver defaults apply when unset): `DATABASE_POOL_MAX`, `DATABASE_IDLE_TIMEOUT_SECONDS`, `DATABASE_CONNECT_TIMEOUT_SECONDS`.+Related optional client tuning: `DATABASE_POOL_MAX`, `DATABASE_IDLE_TIMEOUT_SECONDS`, `DATABASE_CONNECT_TIMEOUT_SECONDS`, `DATABASE_MAX_LIFETIME_SECONDS`, `DATABASE_APPLICATION_NAME`. Driver defaults apply when unset, except that idle pooled connections close after 60 seconds (`DATABASE_IDLE_TIMEOUT_SECONDS=0` keeps them open) and the pool reports `application_name=paperclip`. See [Connection pool settings](#connection-pool-settings).++## Connection Pool Settings++The server opens one postgres.js pool for its own queries (and a second one when `DATABASE_MIGRATION_URL` points at a different connection). Every setting is optional:++| Variable | Default | Effect |+|----------|---------|--------|+| `DATABASE_POOL_MAX` | `10` (driver) | Maximum pooled connections. |+| `DATABASE_IDLE_TIMEOUT_SECONDS` | `60` | Close a pooled connection after this much idle time. `0` keeps idle connections open forever (the driver default). |+| `DATABASE_CONNECT_TIMEOUT_SECONDS` | `30` (driver) | Give up on a connection attempt after this long. |+| `DATABASE_MAX_LIFETIME_SECONDS` | 30–60 min, randomized (driver) | Recycle a pooled connection once it is this old. |+| `DATABASE_APPLICATION_NAME` | `paperclip` | Value of `application_name` in `pg_stat_activity`, so you can find Paperclip's backends: `SELECT * FROM pg_stat_activity WHERE application_name = 'paperclip';` |+| `DATABASE_PREPARED_STATEMENTS` | `true` (driver) | Set `false` behind a transaction-mode pooler (see above). |++The server ends its pools during shutdown (SIGINT/SIGTERM) and when startup fails after the pool was opened, so a restarting server does not leave idle backends behind. Size `max_connections` on the PostgreSQL side for at least `DATABASE_POOL_MAX` per server process plus your other clients.## Switching Between Modesdiff --git a/packages/db/src/client-options.test.ts b/packages/db/src/client-options.test.tsindex 267d14dc17..a6fbce0058 100644--- a/packages/db/src/client-options.test.ts+++ b/packages/db/src/client-options.test.ts@@ -1,5 +1,11 @@import { describe, expect, it } from "vitest";-import { databaseClientOptionsFromEnv, postgresJsOptions } from "./client.js";+import {+ DEFAULT_DATABASE_APPLICATION_NAME,+ DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS,+ databaseClientOptionsFromEnv,+ postgresJsOptions,+ resolveDatabaseClientOptions,+} from "./client.js";describe("databaseClientOptionsFromEnv", () => {it("returns no options when nothing is set, preserving driver defaults", () => {@@ -29,8 +35,20 @@ describe("databaseClientOptionsFromEnv", () => {DATABASE_POOL_MAX: "25",DATABASE_IDLE_TIMEOUT_SECONDS: "60",DATABASE_CONNECT_TIMEOUT_SECONDS: "10",+ DATABASE_MAX_LIFETIME_SECONDS: "1800",+ DATABASE_APPLICATION_NAME: " paperclip-web ",}),- ).toEqual({ maxConnections: 25, idleTimeoutSeconds: 60, connectTimeoutSeconds: 10 });+ ).toEqual({+ maxConnections: 25,+ idleTimeoutSeconds: 60,+ connectTimeoutSeconds: 10,+ maxLifetimeSeconds: 1800,+ applicationName: "paperclip-web",+ });+ });++ it("accepts DATABASE_IDLE_TIMEOUT_SECONDS=0 as an explicit opt-out of idle reaping", () => {+ expect(databaseClientOptionsFromEnv({ DATABASE_IDLE_TIMEOUT_SECONDS: "0" })).toEqual({ idleTimeoutSeconds: 0 });});it("rejects malformed values instead of silently ignoring them", () => {@@ -42,6 +60,18 @@ describe("databaseClientOptionsFromEnv", () => {expect(() => databaseClientOptionsFromEnv({ DATABASE_CONNECT_TIMEOUT_SECONDS: "1.5" })).toThrow(/DATABASE_CONNECT_TIMEOUT_SECONDS/,);+ expect(() => databaseClientOptionsFromEnv({ DATABASE_IDLE_TIMEOUT_SECONDS: "-1" })).toThrow(+ /DATABASE_IDLE_TIMEOUT_SECONDS/,+ );+ expect(() => databaseClientOptionsFromEnv({ DATABASE_IDLE_TIMEOUT_SECONDS: "abc" })).toThrow(+ /DATABASE_IDLE_TIMEOUT_SECONDS/,+ );+ expect(() => databaseClientOptionsFromEnv({ DATABASE_MAX_LIFETIME_SECONDS: "0" })).toThrow(+ /DATABASE_MAX_LIFETIME_SECONDS/,+ );+ expect(() => databaseClientOptionsFromEnv({ DATABASE_MAX_LIFETIME_SECONDS: "NaN" })).toThrow(+ /DATABASE_MAX_LIFETIME_SECONDS/,+ );});it("maps to postgres.js option names", () => {@@ -51,7 +81,42 @@ describe("databaseClientOptionsFromEnv", () => {maxConnections: 25,idleTimeoutSeconds: 60,connectTimeoutSeconds: 10,+ maxLifetimeSeconds: 1800,+ applicationName: "paperclip-web",+ }),+ ).toEqual({+ prepare: false,+ max: 25,+ idle_timeout: 60,+ connect_timeout: 10,+ max_lifetime: 1800,+ connection: { application_name: "paperclip-web" },+ });+ });+});++describe("resolveDatabaseClientOptions", () => {+ it("reaps idle connections and names the pool when the environment sets nothing", () => {+ expect(resolveDatabaseClientOptions({})).toEqual({+ idleTimeoutSeconds: DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS,+ applicationName: DEFAULT_DATABASE_APPLICATION_NAME,+ });+ expect(postgresJsOptions(resolveDatabaseClientOptions(databaseClientOptionsFromEnv({})))).toEqual({+ idle_timeout: DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS,+ connection: { application_name: DEFAULT_DATABASE_APPLICATION_NAME },+ });+ });++ it("keeps every explicit value, including an idle timeout of 0", () => {+ expect(+ resolveDatabaseClientOptions({+ maxConnections: 3,+ idleTimeoutSeconds: 0,+ applicationName: "paperclip-cli",}),- ).toEqual({ prepare: false, max: 25, idle_timeout: 60, connect_timeout: 10 });+ ).toEqual({ maxConnections: 3, idleTimeoutSeconds: 0, applicationName: "paperclip-cli" });+ expect(postgresJsOptions(resolveDatabaseClientOptions({ idleTimeoutSeconds: 0 }))).toMatchObject({+ idle_timeout: 0,+ });});});diff --git a/packages/db/src/client.test.ts b/packages/db/src/client.test.tsindex d5e166890f..52a66676d2 100644--- a/packages/db/src/client.test.ts+++ b/packages/db/src/client.test.ts@@ -3,7 +3,9 @@ import fs from "node:fs";import { afterEach, describe, expect, it } from "vitest";import postgres from "postgres";import {+ DEFAULT_DATABASE_APPLICATION_NAME,applyPendingMigrations,+ createDb,inspectMigrations,resetPostgresDatabase,} from "./client.js";@@ -89,6 +91,48 @@ if (!embeddedPostgresSupport.supported) {);}+describeEmbeddedPostgres("createDb pool defaults", () => {+ it("names its backends and closes them once idle", async () => {+ const url = await createTempDatabase();+ const observer = postgres(url, { max: 1, onnotice: () => {} });+ cleanups.push(async () => {+ await observer.end({ timeout: 1 });+ });++ const backendsNamed = async (name: string) => {+ const rows = await observer`+ select count(*)::int as count from pg_stat_activity where application_name = ${name}+ `;+ return rows[0]?.count ?? 0;+ };++ const db = createDb(url);+ cleanups.push(async () => {+ await db.$client.end({ timeout: 1 });+ });+ const [self] = await db.$client`select application_name from pg_stat_activity where pid = pg_backend_pid()`;+ expect(self?.application_name).toBe(DEFAULT_DATABASE_APPLICATION_NAME);++ const shortLived = createDb(url, { applicationName: "paperclip-idle-test", idleTimeoutSeconds: 1 });+ cleanups.push(async () => {+ await shortLived.$client.end({ timeout: 1 });+ });+ await shortLived.$client`select 1`;+ expect(await backendsNamed("paperclip-idle-test")).toBe(1);++ // The driver closes the idle connection after `idle_timeout`; without the+ // option (the driver default) the backend would stay until the process+ // exits. Wait past the timeout, then poll PostgreSQL's own view.+ const deadline = Date.now() + 10_000;+ let remaining = await backendsNamed("paperclip-idle-test");+ while (remaining > 0 && Date.now() < deadline) {+ await new Promise((resolve) => setTimeout(resolve, 250));+ remaining = await backendsNamed("paperclip-idle-test");+ }+ expect(remaining).toBe(0);+ }, 30_000);+});+describeEmbeddedPostgres("resetPostgresDatabase", () => {it("recreates an existing database so stale tables are removed", async () => {const connectionString = await createTempDatabase();diff --git a/packages/db/src/client.ts b/packages/db/src/client.tsindex 75a988aa88..ed7de6716f 100644--- a/packages/db/src/client.ts+++ b/packages/db/src/client.ts@@ -142,8 +142,32 @@ export interface DatabaseClientOptions {idleTimeoutSeconds?: number;/** postgres.js `connect_timeout` in seconds (driver default: 30). */connectTimeoutSeconds?: number;+ /**+ * postgres.js `max_lifetime` in seconds. Bounds how long one pooled+ * connection is reused before the client replaces it (driver default: a+ * random value between 30 and 60 minutes).+ */+ maxLifetimeSeconds?: number;+ /**+ * postgres.js `connection.application_name`, shown in+ * `pg_stat_activity.application_name`. Lets an operator tell Paperclip's+ * pool apart from other clients of the same database (driver default:+ * `postgres.js`).+ */+ applicationName?: string;}+/**+ * Idle pooled connections close after this many seconds unless+ * `DATABASE_IDLE_TIMEOUT_SECONDS` says otherwise. The driver default keeps an+ * idle connection open forever, so a process that stops issuing queries still+ * holds every backend it ever opened. Set `DATABASE_IDLE_TIMEOUT_SECONDS=0`+ * to restore the driver default.+ */+export const DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS = 60;+/** `application_name` reported to PostgreSQL unless `DATABASE_APPLICATION_NAME` overrides it. */+export const DEFAULT_DATABASE_APPLICATION_NAME = "paperclip";+function envBoolean(env: NodeJS.ProcessEnv, name: string): boolean | undefined {const value = env[name]?.trim().toLowerCase();if (value === undefined || value === "") return undefined;@@ -161,12 +185,28 @@ function envPositiveInteger(env: NodeJS.ProcessEnv, name: string): number | undereturn Number.parseInt(value, 10);}+function envNonNegativeInteger(env: NodeJS.ProcessEnv, name: string): number | undefined {+ const value = env[name]?.trim();+ if (value === undefined || value === "") return undefined;+ if (!/^(?:0|[1-9]\d*)$/.test(value)) {+ throw new Error(`${name} must be a non-negative integer, got: ${env[name]}`);+ }+ return Number.parseInt(value, 10);+}++function envNonEmptyString(env: NodeJS.ProcessEnv, name: string): string | undefined {+ const value = env[name]?.trim();+ if (value === undefined || value === "") return undefined;+ return value;+}+/*** Database client tuning from the environment, so hosted deployments can* adapt to their connection topology (pooled endpoints, network latency)- * without editing source. Every variable is optional; when unset the- * driver defaults apply and behavior is identical to a bare- * `postgres(url)` — self-hosted setups need none of these.+ * without editing source. Every variable is optional. This function returns+ * only the values the environment sets; `resolveDatabaseClientOptions` adds+ * Paperclip's own defaults on top, and the driver defaults apply to the rest+ * — self-hosted setups need none of these.*/export function databaseClientOptionsFromEnv(env: NodeJS.ProcessEnv = process.env): DatabaseClientOptions {const options: DatabaseClientOptions = {};@@ -174,24 +214,48 @@ export function databaseClientOptionsFromEnv(env: NodeJS.ProcessEnv = process.enif (prepare !== undefined) options.prepare = prepare;const maxConnections = envPositiveInteger(env, "DATABASE_POOL_MAX");if (maxConnections !== undefined) options.maxConnections = maxConnections;- const idleTimeoutSeconds = envPositiveInteger(env, "DATABASE_IDLE_TIMEOUT_SECONDS");+ // `0` is allowed here: it disables idle reaping (the driver default).+ const idleTimeoutSeconds = envNonNegativeInteger(env, "DATABASE_IDLE_TIMEOUT_SECONDS");if (idleTimeoutSeconds !== undefined) options.idleTimeoutSeconds = idleTimeoutSeconds;const connectTimeoutSeconds = envPositiveInteger(env, "DATABASE_CONNECT_TIMEOUT_SECONDS");if (connectTimeoutSeconds !== undefined) options.connectTimeoutSeconds = connectTimeoutSeconds;+ const maxLifetimeSeconds = envPositiveInteger(env, "DATABASE_MAX_LIFETIME_SECONDS");+ if (maxLifetimeSeconds !== undefined) options.maxLifetimeSeconds = maxLifetimeSeconds;+ const applicationName = envNonEmptyString(env, "DATABASE_APPLICATION_NAME");+ if (applicationName !== undefined) options.applicationName = applicationName;return options;}+/**+ * Fills in Paperclip's defaults for the options the caller left unset: idle+ * connections are reaped after `DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS`, and the+ * pool identifies itself as `DEFAULT_DATABASE_APPLICATION_NAME`. Everything+ * else stays at the driver default. An explicit value (including+ * `idleTimeoutSeconds: 0`) always wins over the default.+ */+export function resolveDatabaseClientOptions(options: DatabaseClientOptions): DatabaseClientOptions {+ return {+ ...options,+ idleTimeoutSeconds: options.idleTimeoutSeconds ?? DEFAULT_DATABASE_IDLE_TIMEOUT_SECONDS,+ applicationName: options.applicationName ?? DEFAULT_DATABASE_APPLICATION_NAME,+ };+}+export function postgresJsOptions(options: DatabaseClientOptions): Record<string, unknown> {const driverOptions: Record<string, unknown> = {};if (options.prepare !== undefined) driverOptions.prepare = options.prepare;if (options.maxConnections !== undefined) driverOptions.max = options.maxConnections;if (options.idleTimeoutSeconds !== undefined) driverOptions.idle_timeout = options.idleTimeoutSeconds;if (options.connectTimeoutSeconds !== undefined) driverOptions.connect_timeout = options.connectTimeoutSeconds;+ if (options.maxLifetimeSeconds !== undefined) driverOptions.max_lifetime = options.maxLifetimeSeconds;+ if (options.applicationName !== undefined) {+ driverOptions.connection = { application_name: options.applicationName };+ }return driverOptions;}export function createDb(url: string, options?: DatabaseClientOptions) {- const resolved = options ?? databaseClientOptionsFromEnv();+ const resolved = resolveDatabaseClientOptions(options ?? databaseClientOptionsFromEnv());const sql = postgres(url, postgresJsOptions(resolved));const key = hostPortKeyOrNull(url);if (key) registerClient(key, sql);diff --git a/server/src/app.ts b/server/src/app.tsindex 3ca53ecc80..2940a9d665 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -982,6 +982,9 @@ export async function createApp(scheduler.stop();jobCoordinator.stop();disableFeedbackExportFlushes();+ // The scheduler tick queries the database. Stop it here, inside the+ // awaited teardown, so no tick runs after the caller ends the pool.+ scheduler.stop();if (importTransferSweepTimer) {clearInterval(importTransferSweepTimer);importTransferSweepTimer = null;diff --git a/server/src/index.ts b/server/src/index.tsindex a4087526b3..13725d74a3 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -104,6 +104,7 @@ import { conflict } from "./errors.js";import { ensureDecisionSigningSecret } from "./services/decision-signing.js";import { createDecisionRetentionNotifyOriginAgent, createDecisionWakeOriginAgent } from "./services/decision-wakeup.js";import {+ closeHttpListenerForShutdown,coordinateHeartbeatSchedulerShutdown,drainRunExecutionFinalizersForShutdown,finalizeServerShutdown,@@ -158,7 +159,40 @@ export interface StartedServer {shutdown: (signal?: "SIGINT" | "SIGTERM") => Promise<void>;}+// Set by the boot sequence once the primary pool exists. A boot that fails+// after that point (a bootstrap query that throws, for example) must end the+// pool before the caller exits: the driver keeps idle connections open until+// the process dies, and in a restart loop the leftover backends of every+// failed generation can exhaust `max_connections` before the next boot even+// gets a connection.+type StartupDatabaseTeardown = { close: (() => Promise<void>) | null };++// Ends the pool behind a drizzle client. Tolerates a client without `$client`+// (test doubles) and never throws, so it is safe on every exit path.+async function endDatabaseClient(client: unknown, timeoutSeconds: number): Promise<void> {+ const sql = (client as { $client?: { end?: (options?: { timeout?: number }) => Promise<void> } } | null)+ ?.$client;+ if (typeof sql?.end !== "function") return;+ await sql.end({ timeout: timeoutSeconds });+}+export async function startServer(): Promise<StartedServer> {+ const startupDatabase: StartupDatabaseTeardown = { close: null };+ try {+ return await startServerWithDatabaseTeardown(startupDatabase);+ } catch (error) {+ if (startupDatabase.close) {+ await startupDatabase.close().catch((closeError) => {+ logger.error({ err: closeError }, "failed to close database clients after startup failure");+ });+ }+ throw error;+ }+}++async function startServerWithDatabaseTeardown(+ startupDatabase: StartupDatabaseTeardown,+): Promise<StartedServer> {setStartupRecoveryPhase("starting");warnIfUnsupportedNodeVersion(process.versions.node, (message) => logger.warn(message));@@ -586,6 +620,15 @@ export async function startServer(): Promise<StartedServer> {resolvedEmbeddedPostgresPort = port;startupDbInfo = { mode: "embedded-postgres", dataDir, port };}++ // Ends every pool this process opened. Used by the orderly shutdown path+ // (after the application services, before the embedded provider stops) and+ // by the fail-loud startup path, so no exit leaves pooled backends behind.+ const closeDatabaseClients = async () => {+ const clients = pluginMigrationDb === db ? [db] : [db, pluginMigrationDb];+ await Promise.all(clients.map((client) => endDatabaseClient(client, 5)));+ };+ startupDatabase.close = closeDatabaseClients;// A claimed warm-pool stack may restart while its provider environment still// names the pool host. Restore the signed, durable identity before Better@@ -1883,9 +1926,16 @@ export async function startServer(): Promise<StartedServer> {// setup-token login session must stop and release its sandbox lease before// the database and the provider stop, so an orderly shutdown never leaves a// sandbox lease or confidential login state alive past the process exit.+ // The HTTP listener closes first, while every service is still up, so a+ // request in flight is drained against a working server and none reaches+ // a route once the pool is gone; the programmatic close below then finds+ // the listener already closed and skips.await finalizeServerShutdown({signal,shutdownAppServices: appShutdown,+ closeHttpListener: () =>+ closeHttpListenerForShutdown({ server, signal, log: logger }),+ closeDatabase: closeDatabaseClients,stopEmbeddedPostgres,shutdownInstrumentation,shutdownSentry,diff --git a/server/src/shutdown.test.ts b/server/src/shutdown.test.tsindex c3500b3a88..4a4c13bf12 100644--- a/server/src/shutdown.test.ts+++ b/server/src/shutdown.test.ts@@ -1,6 +1,7 @@import { EventEmitter } from "node:events";import { describe, expect, it, vi } from "vitest";import {+ closeHttpListenerForShutdown,coordinateHeartbeatSchedulerShutdown,drainRunExecutionFinalizersForShutdown,finalizeServerShutdown,@@ -32,6 +33,12 @@ describe("finalizeServerShutdown", () => {await release.promise;order.push("appServices:settled");});+ const closeHttpListener = vi.fn(async () => {+ order.push("listener:close");+ });+ const closeDatabase = vi.fn(async () => {+ order.push("database:close");+ });const stopEmbeddedPostgres = vi.fn(async () => {order.push("postgres:stop");});@@ -46,6 +53,8 @@ describe("finalizeServerShutdown", () => {const finalize = finalizeServerShutdown({signal: "SIGTERM",shutdownAppServices,+ closeHttpListener,+ closeDatabase,stopEmbeddedPostgres,shutdownInstrumentation,shutdownSentry,@@ -59,6 +68,10 @@ describe("finalizeServerShutdown", () => {// The cleanup is in flight. The database stop, the instrumentation flush,// and the process exit continuation must all wait for it to settle.await vi.waitFor(() => expect(shutdownAppServices).toHaveBeenCalledOnce());+ // The listener already closed: requests are drained while every service+ // is still available, and nothing after this point can be reached.+ expect(closeHttpListener).toHaveBeenCalledOnce();+ expect(closeDatabase).not.toHaveBeenCalled();expect(stopEmbeddedPostgres).not.toHaveBeenCalled();expect(shutdownInstrumentation).not.toHaveBeenCalled();expect(exited).toBe(false);@@ -68,8 +81,10 @@ describe("finalizeServerShutdown", () => {expect(exited).toBe(true);expect(order).toEqual([+ "listener:close","appServices:start","appServices:settled",+ "database:close","postgres:stop","instrumentation:flush","sentry:flush",@@ -125,6 +140,35 @@ describe("finalizeServerShutdown", () => {expect(exited).toBe(true);});+ it("logs a failed database close and still stops the provider and exits", async () => {+ const order: string[] = [];+ const closeError = new Error("pool end timed out");+ const closeDatabase = vi.fn(async () => {+ order.push("database:close");+ throw closeError;+ });+ const stopEmbeddedPostgres = vi.fn(async () => {+ order.push("postgres:stop");+ });+ const log = stubLogger();++ await finalizeServerShutdown({+ signal: "SIGTERM",+ shutdownAppServices: vi.fn(async () => undefined),+ closeDatabase,+ stopEmbeddedPostgres,+ shutdownInstrumentation: vi.fn(async () => undefined),+ shutdownSentry: vi.fn(async () => undefined),+ log,+ });++ expect(order).toEqual(["database:close", "postgres:stop"]);+ expect(log.error).toHaveBeenCalledWith(+ expect.objectContaining({ err: closeError, signal: "SIGTERM" }),+ "Database client shutdown failed",+ );+ });+it("skips the database stop when no embedded PostgreSQL runs in this process", async () => {const shutdownAppServices = vi.fn(async () => undefined);const shutdownInstrumentation = vi.fn(async () => undefined);@@ -146,6 +190,54 @@ describe("finalizeServerShutdown", () => {});});+describe("closeHttpListenerForShutdown", () => {+ function fakeServer(input: { listening: boolean; closeDelayMs?: number | null }) {+ const closeIdleConnections = vi.fn();+ const closeAllConnections = vi.fn();+ const close = vi.fn((callback?: (err?: Error) => void) => {+ if (input.closeDelayMs === null) return;+ setTimeout(() => callback?.(), input.closeDelayMs ?? 0);+ });+ return { listening: input.listening, close, closeIdleConnections, closeAllConnections };+ }++ it("stops accepting, closes idle keep-alive sockets, and resolves once the listener closed", async () => {+ const server = fakeServer({ listening: true, closeDelayMs: 0 });+ await expect(+ closeHttpListenerForShutdown({ server, signal: "SIGTERM", timeoutMs: 1_000, log: stubLogger() }),+ ).resolves.toBe("closed");+ expect(server.close).toHaveBeenCalledOnce();+ expect(server.closeIdleConnections).toHaveBeenCalledOnce();+ expect(server.closeAllConnections).not.toHaveBeenCalled();+ });++ it("closes the remaining connections when the drain outlives the grace period", async () => {+ vi.useFakeTimers();+ try {+ const server = fakeServer({ listening: true, closeDelayMs: null });+ const log = stubLogger();+ const pending = closeHttpListenerForShutdown({ server, signal: "SIGINT", timeoutMs: 250, log });+ await vi.advanceTimersByTimeAsync(250);+ await expect(pending).resolves.toBe("timed_out");+ expect(server.closeAllConnections).toHaveBeenCalledOnce();+ expect(log.info).toHaveBeenCalledWith(+ expect.objectContaining({ timeoutMs: 250 }),+ expect.stringContaining("timed out"),+ );+ } finally {+ vi.useRealTimers();+ }+ });++ it("does nothing when the listener was never bound", async () => {+ const server = fakeServer({ listening: false });+ await expect(+ closeHttpListenerForShutdown({ server, signal: "SIGTERM", log: stubLogger() }),+ ).resolves.toBe("not_listening");+ expect(server.close).not.toHaveBeenCalled();+ });+});+describe("drainRunExecutionFinalizersForShutdown", () => {it("awaits bounded execution finalizers", async () => {const release = deferred();diff --git a/server/src/shutdown.ts b/server/src/shutdown.tsindex adffd4dc6d..a76501b9f3 100644--- a/server/src/shutdown.ts+++ b/server/src/shutdown.ts@@ -36,6 +36,58 @@ export async function drainRunExecutionFinalizersForShutdown(input: {}}+type ShutdownHttpListener = {+ listening: boolean;+ close(callback?: (err?: Error) => void): unknown;+ closeIdleConnections?: () => void;+ closeAllConnections?: () => void;+};++/**+ * Stops the HTTP listener from accepting new requests and waits, for at most+ * `timeoutMs`, for the open connections to finish. Idle keep-alive sockets+ * close at once; whatever is still open when the grace period ends is closed+ * forcibly, so the teardown never hangs on a long-lived client. Call this+ * before the database pool ends, so no request can reach a route after+ * `sql.end()` and fail with a connection-ended error.+ */+export async function closeHttpListenerForShutdown(input: {+ server: ShutdownHttpListener;+ signal: "SIGINT" | "SIGTERM";+ timeoutMs?: number;+ log: ShutdownLogger;+}): Promise<"closed" | "timed_out" | "not_listening"> {+ if (!input.server.listening) return "not_listening";+ const timeoutMs = input.timeoutMs ?? 5_000;+ let timer: NodeJS.Timeout | null = null;+ try {+ return await Promise.race([+ new Promise<"closed">((resolve) => {+ input.server.close((err) => {+ if (err && (err as NodeJS.ErrnoException).code !== "ERR_SERVER_NOT_RUNNING") {+ input.log.error({ err, signal: input.signal }, "HTTP listener close failed");+ }+ resolve("closed");+ });+ input.server.closeIdleConnections?.();+ }),+ new Promise<"timed_out">((resolve) => {+ timer = setTimeout(() => {+ input.log.info(+ { signal: input.signal, timeoutMs },+ "HTTP listener drain timed out; closing the remaining connections",+ );+ input.server.closeAllConnections?.();+ resolve("timed_out");+ }, timeoutMs);+ timer.unref?.();+ }),+ ]);+ } finally {+ if (timer) clearTimeout(timer);+ }+}+/*** Runs the final, ordered teardown of the server. It awaits the application* service cleanup first, so a live setup-token login session stops and releases@@ -52,6 +104,20 @@ export async function drainRunExecutionFinalizersForShutdown(input: {export async function finalizeServerShutdown(input: {signal: "SIGINT" | "SIGTERM";shutdownAppServices: (() => Promise<void>) | undefined;+ /**+ * Stops the HTTP listener and drains its connections (see+ * `closeHttpListenerForShutdown`). Runs first, while every application+ * service is still available to the requests being drained, so no request+ * runs against a half-dismantled service or an ended pool.+ */+ closeHttpListener?: (() => Promise<unknown>) | null;+ /**+ * Ends the server's PostgreSQL client pools. Runs after the application+ * services (which still need the database) and before the embedded+ * provider stops, so the backends close in order and none outlive the+ * process.+ */+ closeDatabase?: (() => Promise<void>) | null;stopEmbeddedPostgres: (() => Promise<void>) | null;shutdownInstrumentation: () => Promise<void>;shutdownSentry: () => Promise<void>;@@ -59,6 +125,16 @@ export async function finalizeServerShutdown(input: {}): Promise<void> {const { signal } = input;+ // Stop accepting requests and drain the open ones before any service goes+ // away, so a request that is still in flight sees a fully working server.+ if (input.closeHttpListener) {+ try {+ await input.closeHttpListener();+ } catch (err) {+ input.log.error({ err, signal }, "HTTP listener shutdown failed");+ }+ }+// Await the application service cleanup, so a live setup-token login session// releases its sandbox lease before the database and the provider stop. A// rejected cleanup stays durable for the reaper; it does not block the exit.@@ -68,6 +144,18 @@ export async function finalizeServerShutdown(input: {input.log.error({ err, signal }, "Application service shutdown failed");}+ // End the client pools once nothing needs them any more. Without this the+ // process exit leaves the pooled backends to PostgreSQL's own TCP keepalive+ // reaping, and a restart loop can pile up enough of them to hit+ // `max_connections` before the next boot gets a connection.+ if (input.closeDatabase) {+ try {+ await input.closeDatabase();+ } catch (err) {+ input.log.error({ err, signal }, "Database client shutdown failed");+ }+ }+if (input.stopEmbeddedPostgres) {input.log.info({ signal }, "Stopping embedded PostgreSQL");try {