Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep Postgres pools warm per deployment profile to stop pooler login storms (#4495)
Gergő Móriczc9f94b3afbd4
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 1e69032a3..83c4745da 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -8,6 +8,9 @@ PLAYWRIGHT_MICROSERVICE_URL=http://playwright-service:3000/scrape## To turn on DB authentication, you need to set up supabase.USE_DB_AUTHENTICATION=true+# Postgres pool sizing preset for this process: api | worker | utility (see src/db/pool-profiles.ts).+# Leave unset to keep the default pool settings.+# DB_POOL_PROFILE=# ===== Optional ENVS ======diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 129532cf8..72cb20029 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -197,6 +197,10 @@ const configSchema = z.object({DATABASE_URL: z.string().optional(),DATABASE_REPLICA_URL: z.string().optional(),INDEX_DATABASE_URL: z.string().optional(),+ // Pool sizing preset for this process (see db/pool-profiles.ts). Unset keeps+ // the historical pool settings; deployments opt into `api`, `worker` or+ // `utility` to keep connections warm within the pooler's client budget.+ DB_POOL_PROFILE: emptyStringAsUndefined(z.enum(["api", "worker", "utility"])),INDEX_CACHE_REDIS_URL: z.string().optional(),// Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables// it; the cache then only shields lookups that find data. A positive valuediff --git a/apps/api/src/db/connection.ts b/apps/api/src/db/connection.tsindex 64438803f..e859cd83e 100644--- a/apps/api/src/db/connection.ts+++ b/apps/api/src/db/connection.ts@@ -2,42 +2,70 @@ import { Pool } from "pg";import { drizzle, NodePgDatabase } from "drizzle-orm/node-postgres";import { config } from "../config";import { logger } from "../lib/logger";+import {+ keepPoolWarm,+ pickMaxLifetimeSeconds,+ resolveDbPoolOptions,+ type DbPoolName,+} from "./pool-profiles";type DB = NodePgDatabase;-interface PoolSizing {- /** Max client connections this pool opens to the pooler. Default 20. */- max?: number;- /** Idle connections held open permanently. Default 0. */- min?: number;-}-// Registry of every pool we create, for /metrics exposure.const dbPools: { name: string; pool: Pool }[] = [];+// One lifetime for every pool in this process; randomized per process so+// recycling never lines up across the fleet (see pool-profiles.ts).+const poolMaxLifetimeSeconds = pickMaxLifetimeSeconds();+function makeDb(connectionString: string | undefined,applicationName: string,- sizing: PoolSizing = {},+ poolName: DbPoolName,+ { keepWarm = true }: { keepWarm?: boolean } = {},): DB | null {if (!connectionString) {return null;}+ // Sizing comes from the deployment's DB_POOL_PROFILE (pool-profiles.ts).+ // Each process opens up to `max` client connections per pool against the+ // (transaction) pooler. Supabase pins pgbouncer's max_client_conn at 12000,+ // so the fleet-wide budget is `pods * max` per pool, and per-process pools+ // stay small: the transaction pooler multiplexes server connections, so a+ // large client pool buys little throughput but eats the global cap. `min`+ // keeps connections warm across traffic dips so a burst never has the whole+ // fleet re-opening connections at once.+ const options = resolveDbPoolOptions(+ config.DB_POOL_PROFILE,+ poolName,+ poolMaxLifetimeSeconds,+ );+ // A pool that is not kept warm keeps the profile's connect timeout,+ // lifetime and keepalive but drains like before (no floor, 10 s idle+ // eviction) so it never double-counts against the pooler's client budget.+ if (!keepWarm) {+ options.min = 0;+ options.idleTimeoutMillis = 10_000;+ }const pool = new Pool({connectionString,application_name: applicationName,- // Each process opens up to `max` client connections per pool against the- // (transaction) pooler. Supabase pins pgbouncer's max_client_conn at 12000,- // so the fleet-wide budget is `pods * sum(max across pools)`. Keep these- // small — the transaction pooler multiplexes server connections, so a large- // per-process client pool buys little throughput but eats the global cap.- // `min: 0` lets idle pods release connections instead of holding them.- max: sizing.max ?? 20,- min: sizing.min ?? 0,+ max: options.max,+ min: options.min,+ idleTimeoutMillis: options.idleTimeoutMillis,+ connectionTimeoutMillis: options.connectionTimeoutMillis,+ maxLifetimeSeconds: options.maxLifetimeSeconds,keepAlive: true,+ keepAliveInitialDelayMillis: options.keepAliveInitialDelayMillis,});dbPools.push({ name: applicationName, pool });+ logger.info("Postgres pool configured", {+ module: "db",+ applicationName,+ profile: config.DB_POOL_PROFILE ?? "default",+ ...options,+ });pool.on("error", err =>logger.error("Error in idle Postgres client", {err,@@ -66,6 +94,17 @@ function makeDb(}});+ keepPoolWarm(pool, options.min, {+ initialDelayMs: Math.floor(Math.random() * 60_000),+ intervalMs: 5_000 + Math.floor(Math.random() * 2_000),+ onError: error =>+ logger.warn("Postgres pool warm-up connect failed", {+ module: "db",+ applicationName,+ error,+ }),+ });+return drizzle({ client: pool });}@@ -107,28 +146,43 @@ export function getDbPoolMetrics(): string {`db_pool_max_count{application_name="${name}"} ${pool.options.max ?? 20}`,);}+ lines.push(+ "# HELP db_pool_min_count Configured minimum (kept-warm) connections for this pool",+ "# TYPE db_pool_min_count gauge",+ );+ for (const { name, pool } of dbPools) {+ lines.push(+ `db_pool_min_count{application_name="${name}"} ${pool.options.min ?? 0}`,+ );+ }return lines.join("\n");}const useDbAuthentication = config.USE_DB_AUTHENTICATION;const mainDb = useDbAuthentication- ? makeDb(config.DATABASE_URL, "firecrawl-api")+ ? makeDb(config.DATABASE_URL, "firecrawl-api", "main"): null;+// Without a distinct replica URL the "replica" pool points at the primary's+// pooler, so it must not hold a second warm set of connections there.const replicaDb = useDbAuthentication? makeDb(config.DATABASE_REPLICA_URL ?? config.DATABASE_URL,"firecrawl-api-rr",+ "replica",+ {+ keepWarm:+ config.DATABASE_REPLICA_URL !== undefined &&+ config.DATABASE_REPLICA_URL !== config.DATABASE_URL,+ },): null;// The index pool was the sole consumer behind the 2026-06-11 pgbouncer// `08P01: no more connections allowed (max_client_conn)` incident. It runs-// against the transaction pooler, so cap it well below the generic 20 to keep-// the fleet-wide client-connection count under Supabase's 12000 ceiling.-const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index", {- max: 6,- min: 0,-});+// against the transaction pooler, so every profile caps it well below the+// generic pools to keep the fleet-wide client-connection count under+// Supabase's 12000 ceiling.+const indexDb = makeDb(config.INDEX_DATABASE_URL, "firecrawl-index", "index");if (useDbAuthentication && !mainDb) {logger.error(diff --git a/apps/api/src/db/pool-profiles.ts b/apps/api/src/db/pool-profiles.tsnew file mode 100644index 000000000..3df298a35--- /dev/null+++ b/apps/api/src/db/pool-profiles.ts@@ -0,0 +1,202 @@+/**+ * Postgres pool sizing presets, selected per deployment with `DB_POOL_PROFILE`.+ *+ * Every process opens its own node-pg pools against the transaction pooler.+ * With `min: 0` and pg-pool's default 10 s idle timeout, a pool drains to+ * nothing whenever traffic dips and re-opens all of its connections at once+ * when traffic jumps. Hundreds of pods doing that in the same second is a+ * login storm (a TLS handshake and an auth query per connection) that stalls+ * the pooler for every query behind it while Postgres itself sits idle; with+ * top-of-the-hour cron traffic it turns into a rhythmic latency spike.+ *+ * A profile balances the two things the pooler cares about:+ * - warm connections per pod (`min`, kept topped up by `keepPoolWarm`),+ * which remove the synchronized reconnects, against+ * - the fleet-wide client budget (`pods × max` per pool has to stay under+ * the pooler's max_client_conn), which is why horizontally scaled workers+ * get tiny pools and only the API tier keeps large warm ones.+ *+ * Unset keeps the historical settings so deploying the code alone changes+ * nothing; the profile is switched on per deployment.+ */++type DbPoolProfile = "api" | "worker" | "utility";+export type DbPoolName = "main" | "replica" | "index";++interface DbPoolOptions {+ max: number;+ /**+ * Warm floor. pg-pool only skips the idle timer for a client released while+ * the pool is at or below `min`; it neither pre-opens connections nor refills+ * after evictions, so the floor is actually held by `keepPoolWarm`.+ */+ min: number;+ /** 0 disables idle eviction. */+ idleTimeoutMillis: number;+ /** Bounds both the connect handshake and the wait for a free slot; 0 waits forever. */+ connectionTimeoutMillis: number;+ /** 0 disables lifetime-based recycling. */+ maxLifetimeSeconds: number;+ /** undefined leaves TCP keepalive probing at the OS default (hours on Linux). */+ keepAliveInitialDelayMillis: number | undefined;+}++type PoolSize = { max: number; min: number };++/** Historical sizes, used when no profile is set. */+const DEFAULT_MAX: Record<DbPoolName, number> = {+ main: 20,+ replica: 20,+ index: 6,+};++const PROFILE_SIZES: Record<DbPoolProfile, Record<DbPoolName, PoolSize>> = {+ // API tier: a few hundred pods that see the traffic bursts first. Fully warm+ // pools so a burst never opens a connection; the budget is pods × 20.+ api: {+ main: { max: 20, min: 20 },+ replica: { max: 20, min: 20 },+ index: { max: 6, min: 6 },+ },+ // Horizontally scaled job workers (potentially thousands of pods). Each pod+ // does a handful of short writes per minute, so one warm connection covers+ // it; the replica and index pools are only touched at boot or on cache+ // misses and may drain.+ worker: {+ main: { max: 2, min: 1 },+ replica: { max: 2, min: 0 },+ index: { max: 2, min: 0 },+ },+ // Small fixed-size or moderately scaled deployments (schedulers, indexers,+ // janitors) that run several DB-bound loops concurrently.+ utility: {+ main: { max: 4, min: 1 },+ replica: { max: 2, min: 0 },+ index: { max: 6, min: 0 },+ },+};++const CONNECT_TIMEOUT_MS: Record<DbPoolProfile, number> = {+ api: 15_000,+ worker: 30_000,+ utility: 30_000,+};++/** Connections above `min` linger long enough to outlast a burst and its aftershocks. */+const WARM_IDLE_TIMEOUT_MS = 10 * 60_000;+const KEEPALIVE_DELAY_MS = 30_000;++/**+ * Lifetime recycling is randomized per process between 4 h and 8 h. A short or+ * fixed lifetime would expire every connection a burst opened at the same+ * moment, fleet-wide, just before the next burst — re-creating the cold-pool+ * problem on a schedule.+ */+const MAX_LIFETIME_MIN_SECONDS = 4 * 3600;+const MAX_LIFETIME_SPREAD_SECONDS = 4 * 3600;++export function pickMaxLifetimeSeconds(): number {+ return (+ MAX_LIFETIME_MIN_SECONDS ++ Math.floor(Math.random() * MAX_LIFETIME_SPREAD_SECONDS)+ );+}++export function resolveDbPoolOptions(+ profile: DbPoolProfile | undefined,+ pool: DbPoolName,+ maxLifetimeSeconds: number,+): DbPoolOptions {+ if (profile === undefined) {+ return {+ max: DEFAULT_MAX[pool],+ min: 0,+ idleTimeoutMillis: 10_000,+ connectionTimeoutMillis: 0,+ maxLifetimeSeconds: 0,+ keepAliveInitialDelayMillis: undefined,+ };+ }+ const size = PROFILE_SIZES[profile][pool];+ return {+ max: size.max,+ min: size.min,+ idleTimeoutMillis: WARM_IDLE_TIMEOUT_MS,+ connectionTimeoutMillis: CONNECT_TIMEOUT_MS[profile],+ maxLifetimeSeconds,+ keepAliveInitialDelayMillis: KEEPALIVE_DELAY_MS,+ };+}++/** The slice of pg.Pool the warm-up loop needs. */+interface WarmablePool {+ readonly totalCount: number;+ readonly idleCount: number;+ readonly ending: boolean;+ connect(): Promise<{ release(): void }>;+}++interface KeepWarmOptions {+ /** Delay before the first attempt, so pods started together don't log in together. */+ initialDelayMs: number;+ intervalMs: number;+ onError: (error: unknown) => void;+}++/**+ * Tops a pool up to `min` open connections, one per tick.+ *+ * pg-pool never opens connections ahead of demand, and `min` only suppresses+ * the idle timer for clients released while the pool is at or below it (a+ * client released above `min` is evicted later even if the pool has shrunk+ * since), so a fresh pod, a recycled connection or a drained worker pool would+ * otherwise do its logins in the first burst it serves. This loop pays that+ * cost gradually instead. `connect()` hands out idle clients before opening+ * new ones, so a tick briefly checks the idle ones out to force exactly one+ * new connection and hands them back as soon as it is registered. Returns a+ * function that stops the loop.+ */+export function keepPoolWarm(+ pool: WarmablePool,+ min: number,+ options: KeepWarmOptions,+): () => void {+ if (min <= 0) return () => {};++ let inFlight = false;+ const tick = async () => {+ if (inFlight || pool.ending || pool.totalCount >= min) return;+ inFlight = true;+ const held: { release(): void }[] = [];+ try {+ while (pool.idleCount > 0 && pool.totalCount < min) {+ held.push(await pool.connect());+ }+ if (pool.totalCount < min) {+ // pg-pool registers the new client synchronously inside connect(), so+ // the idle ones can go straight back: real queries never wait behind+ // this handshake, and the pool cannot refill past `min`.+ const opening = pool.connect();+ for (const client of held.splice(0)) client.release();+ held.push(await opening);+ }+ } catch (error) {+ options.onError(error);+ } finally {+ for (const client of held) client.release();+ inFlight = false;+ }+ };++ let interval: NodeJS.Timeout | undefined;+ const start = setTimeout(() => {+ interval = setInterval(() => void tick(), options.intervalMs);+ interval.unref();+ }, options.initialDelayMs);+ start.unref();++ return () => {+ clearTimeout(start);+ if (interval) clearInterval(interval);+ };+}