Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): keep Postgres pools warm per deployment profile to stop pooler login storms (#4495)

Gergő Móriczc9f94b3afbd4
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 1e69032a3..83c4745da 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -8,6 +8,9 @@ PLAYWRIGHT_MICROSERVICE_URL=http://playwright-service:3000/scrape
## To turn on DB authentication, you need to set up supabase.
USE_DB_AUTHENTICATION=true
+# Postgres pool sizing preset for this process: api | worker | utility (see src/db/pool-profiles.ts).
+# Leave unset to keep the default pool settings.
+# DB_POOL_PROFILE=
# ===== Optional ENVS ======
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 129532cf8..72cb20029 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -197,6 +197,10 @@ const configSchema = z.object({
DATABASE_URL: z.string().optional(),
DATABASE_REPLICA_URL: z.string().optional(),
INDEX_DATABASE_URL: z.string().optional(),
+ // Pool sizing preset for this process (see db/pool-profiles.ts). Unset keeps
+ // the historical pool settings; deployments opt into `api`, `worker` or
+ // `utility` to keep connections warm within the pooler's client budget.
+ DB_POOL_PROFILE: emptyStringAsUndefined(z.enum(["api", "worker", "utility"])),
INDEX_CACHE_REDIS_URL: z.string().optional(),
// Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables
// it; the cache then only shields lookups that find data. A positive value
diff --git a/apps/api/src/db/connection.ts b/apps/api/src/db/connection.ts
index 64438803f..e859cd83e 100644
--- a/apps/api/src/db/connection.ts
+++ b/apps/api/src/db/connection.ts
@@ -2,42 +2,70 @@ import { Pool } from "pg";
import { drizzle, NodePgDatabase } from "drizzle-orm/node-postgres";
import { config } from "../config";
import { logger } from "../lib/logger";
+import {
+ keepPoolWarm,
+ pickMaxLifetimeSeconds,
+ resolveDbPoolOptions,
+ type DbPoolName,
+} from "./pool-profiles";
type DB = NodePgDatabase;
-interface PoolSizing {
- /** Max client connections this pool opens to the pooler. Default 20. */
- max?: number;
- /** Idle connections held open permanently. Default 0. */
- min?: number;
-}
-
// Registry of every pool we create, for /metrics exposure.
const dbPools: { name: string; pool: Pool }[] = [];
+// One lifetime for every pool in this process; randomized per process so
+// recycling never lines up across the fleet (see pool-profiles.ts).
+const poolMaxLifetimeSeconds = pickMaxLifetimeSeconds();
+
function makeDb(
connectionString: string | undefined,
applicationName: string,
- sizing: PoolSizing = {},
+ poolName: DbPoolName,
+ { keepWarm = true }: { keepWarm?: boolean } = {},
): DB | null {
if (!connectionString) {
return null;
}
+ // Sizing comes from the deployment's DB_POOL_PROFILE (pool-profiles.ts).
+ // Each process opens up to `max` client connections per pool against the
+ // (transaction) pooler. Supabase pins pgbouncer's max_client_conn at 12000,
+ // so the fleet-wide budget is `pods * max` per pool, and per-process pools
+ // stay small: the transaction pooler multiplexes server connections, so a
+ // large client pool buys little throughput but eats the global cap. `min`
+ // keeps connections warm across traffic dips so a burst never has the whole
+ // fleet re-opening connections at once.
+ const options = resolveDbPoolOptions(
+ config.DB_POOL_PROFILE,
+ poolName,
+ poolMaxLifetimeSeconds,
+ );
+ // A pool that is not kept warm keeps the profile's connect timeout,
+ // lifetime and keepalive but drains like before (no floor, 10 s idle
+ // eviction) so it never double-counts against the pooler's client budget.
+ if (!keepWarm) {
+ options.min = 0;
+ options.idleTimeoutMillis = 10_000;
+ }
const pool = new Pool({
connectionString,
application_name: applicationName,
- // Each process opens up to `max` client connections per pool against the
- // (transaction) pooler. Supabase pins pgbouncer's max_client_conn at 12000,
- // so the fleet-wide budget is `pods * sum(max across pools)`. Keep these
- // small — the transaction pooler multiplexes server connections, so a large
- // per-process client pool buys little throughput but eats the global cap.
- // `min: 0` lets idle pods release connections instead of holding them.
- max: sizing.max ?? 20,
- min: sizing.min ?? 0,
+ max: options.max,
+ min: options.min,
+ idleTimeoutMillis: options.idleTimeoutMillis,
+ connectionTimeoutMillis: options.connectionTimeoutMillis,
+ maxLifetimeSeconds: options.maxLifetimeSeconds,
keepAlive: true,
+ keepAliveInitialDelayMillis: options.keepAliveInitialDelayMillis,
});
dbPools.push({ name: applicationName, pool });
+ logger.info("Postgres pool configured", {
+ module: "db",
+ applicationName,
+ profile: config.DB_POOL_PROFILE ?? "default",
+ ...options,
+ });
pool.on("error", err =>
logger.error("Error in idle Postgres client", {
err,
@@ -66,6 +94,17 @@ function makeDb(
}
});
+ keepPoolWarm(pool, options.min, {
+ initialDelayMs: Math.floor(Math.random() * 60_000),
+ intervalMs: 5_000 + Math.floor(Math.random() * 2_000),
+ onError: error =>
+ logger.warn("Postgres pool warm-up connect failed", {
+ module: "db",
+ applicationName,
+ error,
+ }),
+ });
+
return drizzle({ client: pool });
}
@@ -107,28 +146,43 @@ export function getDbPoolMetrics(): string {
`db_pool_max_count{application_name="${name}"} ${pool.options.max ?? 20}`,
);
}
+ lines.push(
+ "# HELP db_pool_min_count Configured minimum (kept-warm) connections for this pool",
+ "# TYPE db_pool_min_count gauge",
+ );
+ for (const { name, pool } of dbPools) {
+ lines.push(
+ `db_pool_min_count{application_name="${name}"} ${pool.options.min ?? 0}`,
+ );
+ }
return lines.join("\n");
}
const useDbAuthentication = config.USE_DB_AUTHENTICATION;
const mainDb = useDbAuthentication
- ? makeDb(config.DATABASE_URL, "firecrawl-api")
+ ? makeDb(config.DATABASE_URL, "firecrawl-api", "main")
: null;
+// Without a distinct replica URL the "replica" pool points at the primary's
+// pooler, so it must not hold a second warm set of connections there.
const replicaDb = useDbAuthentication
? makeDb(
config.DATABASE_REPLICA_URL ?? config.DATABASE_URL,
"firecrawl-api-rr",
+ "replica",
+ {
+ keepWarm:
+ config.DATABASE_REPLICA_URL !== undefined &&
+ config.DATABASE_REPLICA_URL !== config.DATABASE_URL,
+ },
)
: null;
// The index pool was the sole consumer behind the 2026-06-11 pgbouncer
// `08P01: no more connections allowed (max_client_conn)` incident. It runs
-// against the transaction pooler, so cap it well below the generic 20 to keep
-// the fleet-wide client-connection count under Supabase's 12000 ceiling.
-const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index", {
- max: 6,
- min: 0,
-});
+// against the transaction pooler, so every profile caps it well below the
+// generic pools to keep the fleet-wide client-connection count under
+// Supabase's 12000 ceiling.
+const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index", "index");
if (useDbAuthentication && !mainDb) {
logger.error(
diff --git a/apps/api/src/db/pool-profiles.ts b/apps/api/src/db/pool-profiles.ts
new file mode 100644
index 000000000..3df298a35
--- /dev/null
+++ b/apps/api/src/db/pool-profiles.ts
@@ -0,0 +1,202 @@
+/**
+ * Postgres pool sizing presets, selected per deployment with `DB_POOL_PROFILE`.
+ *
+ * Every process opens its own node-pg pools against the transaction pooler.
+ * With `min: 0` and pg-pool's default 10 s idle timeout, a pool drains to
+ * nothing whenever traffic dips and re-opens all of its connections at once
+ * when traffic jumps. Hundreds of pods doing that in the same second is a
+ * login storm (a TLS handshake and an auth query per connection) that stalls
+ * the pooler for every query behind it while Postgres itself sits idle; with
+ * top-of-the-hour cron traffic it turns into a rhythmic latency spike.
+ *
+ * A profile balances the two things the pooler cares about:
+ * - warm connections per pod (`min`, kept topped up by `keepPoolWarm`),
+ * which remove the synchronized reconnects, against
+ * - the fleet-wide client budget (`pods × max` per pool has to stay under
+ * the pooler's max_client_conn), which is why horizontally scaled workers
+ * get tiny pools and only the API tier keeps large warm ones.
+ *
+ * Unset keeps the historical settings so deploying the code alone changes
+ * nothing; the profile is switched on per deployment.
+ */
+
+type DbPoolProfile = "api" | "worker" | "utility";
+export type DbPoolName = "main" | "replica" | "index";
+
+interface DbPoolOptions {
+ max: number;
+ /**
+ * Warm floor. pg-pool only skips the idle timer for a client released while
+ * the pool is at or below `min`; it neither pre-opens connections nor refills
+ * after evictions, so the floor is actually held by `keepPoolWarm`.
+ */
+ min: number;
+ /** 0 disables idle eviction. */
+ idleTimeoutMillis: number;
+ /** Bounds both the connect handshake and the wait for a free slot; 0 waits forever. */
+ connectionTimeoutMillis: number;
+ /** 0 disables lifetime-based recycling. */
+ maxLifetimeSeconds: number;
+ /** undefined leaves TCP keepalive probing at the OS default (hours on Linux). */
+ keepAliveInitialDelayMillis: number | undefined;
+}
+
+type PoolSize = { max: number; min: number };
+
+/** Historical sizes, used when no profile is set. */
+const DEFAULT_MAX: Record<DbPoolName, number> = {
+ main: 20,
+ replica: 20,
+ index: 6,
+};
+
+const PROFILE_SIZES: Record<DbPoolProfile, Record<DbPoolName, PoolSize>> = {
+ // API tier: a few hundred pods that see the traffic bursts first. Fully warm
+ // pools so a burst never opens a connection; the budget is pods × 20.
+ api: {
+ main: { max: 20, min: 20 },
+ replica: { max: 20, min: 20 },
+ index: { max: 6, min: 6 },
+ },
+ // Horizontally scaled job workers (potentially thousands of pods). Each pod
+ // does a handful of short writes per minute, so one warm connection covers
+ // it; the replica and index pools are only touched at boot or on cache
+ // misses and may drain.
+ worker: {
+ main: { max: 2, min: 1 },
+ replica: { max: 2, min: 0 },
+ index: { max: 2, min: 0 },
+ },
+ // Small fixed-size or moderately scaled deployments (schedulers, indexers,
+ // janitors) that run several DB-bound loops concurrently.
+ utility: {
+ main: { max: 4, min: 1 },
+ replica: { max: 2, min: 0 },
+ index: { max: 6, min: 0 },
+ },
+};
+
+const CONNECT_TIMEOUT_MS: Record<DbPoolProfile, number> = {
+ api: 15_000,
+ worker: 30_000,
+ utility: 30_000,
+};
+
+/** Connections above `min` linger long enough to outlast a burst and its aftershocks. */
+const WARM_IDLE_TIMEOUT_MS = 10 * 60_000;
+const KEEPALIVE_DELAY_MS = 30_000;
+
+/**
+ * Lifetime recycling is randomized per process between 4 h and 8 h. A short or
+ * fixed lifetime would expire every connection a burst opened at the same
+ * moment, fleet-wide, just before the next burst — re-creating the cold-pool
+ * problem on a schedule.
+ */
+const MAX_LIFETIME_MIN_SECONDS = 4 * 3600;
+const MAX_LIFETIME_SPREAD_SECONDS = 4 * 3600;
+
+export function pickMaxLifetimeSeconds(): number {
+ return (
+ MAX_LIFETIME_MIN_SECONDS +
+ Math.floor(Math.random() * MAX_LIFETIME_SPREAD_SECONDS)
+ );
+}
+
+export function resolveDbPoolOptions(
+ profile: DbPoolProfile | undefined,
+ pool: DbPoolName,
+ maxLifetimeSeconds: number,
+): DbPoolOptions {
+ if (profile === undefined) {
+ return {
+ max: DEFAULT_MAX[pool],
+ min: 0,
+ idleTimeoutMillis: 10_000,
+ connectionTimeoutMillis: 0,
+ maxLifetimeSeconds: 0,
+ keepAliveInitialDelayMillis: undefined,
+ };
+ }
+ const size = PROFILE_SIZES[profile][pool];
+ return {
+ max: size.max,
+ min: size.min,
+ idleTimeoutMillis: WARM_IDLE_TIMEOUT_MS,
+ connectionTimeoutMillis: CONNECT_TIMEOUT_MS[profile],
+ maxLifetimeSeconds,
+ keepAliveInitialDelayMillis: KEEPALIVE_DELAY_MS,
+ };
+}
+
+/** The slice of pg.Pool the warm-up loop needs. */
+interface WarmablePool {
+ readonly totalCount: number;
+ readonly idleCount: number;
+ readonly ending: boolean;
+ connect(): Promise<{ release(): void }>;
+}
+
+interface KeepWarmOptions {
+ /** Delay before the first attempt, so pods started together don't log in together. */
+ initialDelayMs: number;
+ intervalMs: number;
+ onError: (error: unknown) => void;
+}
+
+/**
+ * Tops a pool up to `min` open connections, one per tick.
+ *
+ * pg-pool never opens connections ahead of demand, and `min` only suppresses
+ * the idle timer for clients released while the pool is at or below it (a
+ * client released above `min` is evicted later even if the pool has shrunk
+ * since), so a fresh pod, a recycled connection or a drained worker pool would
+ * otherwise do its logins in the first burst it serves. This loop pays that
+ * cost gradually instead. `connect()` hands out idle clients before opening
+ * new ones, so a tick briefly checks the idle ones out to force exactly one
+ * new connection and hands them back as soon as it is registered. Returns a
+ * function that stops the loop.
+ */
+export function keepPoolWarm(
+ pool: WarmablePool,
+ min: number,
+ options: KeepWarmOptions,
+): () => void {
+ if (min <= 0) return () => {};
+
+ let inFlight = false;
+ const tick = async () => {
+ if (inFlight || pool.ending || pool.totalCount >= min) return;
+ inFlight = true;
+ const held: { release(): void }[] = [];
+ try {
+ while (pool.idleCount > 0 && pool.totalCount < min) {
+ held.push(await pool.connect());
+ }
+ if (pool.totalCount < min) {
+ // pg-pool registers the new client synchronously inside connect(), so
+ // the idle ones can go straight back: real queries never wait behind
+ // this handshake, and the pool cannot refill past `min`.
+ const opening = pool.connect();
+ for (const client of held.splice(0)) client.release();
+ held.push(await opening);
+ }
+ } catch (error) {
+ options.onError(error);
+ } finally {
+ for (const client of held) client.release();
+ inFlight = false;
+ }
+ };
+
+ let interval: NodeJS.Timeout | undefined;
+ const start = setTimeout(() => {
+ interval = setInterval(() => void tick(), options.intervalMs);
+ interval.unref();
+ }, options.initialDelayMs);
+ start.unref();
+
+ return () => {
+ clearTimeout(start);
+ if (interval) clearInterval(interval);
+ };
+}