:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(ingestion): fail startup when a redis pool cannot connect (#122654)
Paweł Ledwoń1a2d4398b712
diff --git a/nodejs/src/common/utils/db/redis-pool-component.test.ts b/nodejs/src/common/utils/db/redis-pool-component.test.tsnew file mode 100644index 000000000000..0dfa1cf26baf--- /dev/null+++ b/nodejs/src/common/utils/db/redis-pool-component.test.ts@@ -0,0 +1,34 @@+import { RedisConnectionConfig, RedisPoolComponent } from './redis'++const giveUpBeforeErrorLimit = (attempt: number): number | null => (attempt < 5 ? 25 : null)++const unreachableRedis = (overrides: Partial<RedisConnectionConfig>): RedisConnectionConfig => ({+ url: '127.0.0.1',+ options: { port: 1, commandTimeout: 100, retryStrategy: giveUpBeforeErrorLimit },+ name: 'unreachable-test',+ ...overrides,+})++describe('RedisPoolComponent', () => {+ it('fails to start when Redis is unreachable', async () => {+ const component = new RedisPoolComponent({+ connection: unreachableRedis({}),+ poolMinSize: 1,+ poolMaxSize: 1,+ acquireTimeoutMillis: 200,+ })++ await expect(component.start()).rejects.toThrow()+ })++ it('starts a non-fatal pool when Redis is unreachable', async () => {+ const component = new RedisPoolComponent({+ connection: unreachableRedis({ exitOnRepeatedErrors: false }),+ poolMinSize: 1,+ poolMaxSize: 1,+ })++ const started = await component.start()+ await started.stop()+ })+})diff --git a/nodejs/src/common/utils/db/redis.ts b/nodejs/src/common/utils/db/redis.tsindex f6aa01137904..839cd89cef7e 100644--- a/nodejs/src/common/utils/db/redis.ts+++ b/nodejs/src/common/utils/db/redis.ts@@ -7,15 +7,10 @@ import { logger } from '../../utils/logger'import { killGracefully } from '../../utils/utils'import { captureException } from '../posthog'-/** Number of Redis error events until the server is killed gracefully. */+/** Redis errors a fatal client tolerates. The next error stops the process. */const REDIS_ERROR_COUNTER_LIMIT = 10const NON_FATAL_CONNECT_TIMEOUT_MS = 10_000-/**- * Configuration for a Redis connection.- * Consumers should build this config inline where they create Redis connections,- * rather than relying on centralized builder functions.- */export interface RedisConnectionConfig {url: stringoptions?: RedisOptions@@ -28,9 +23,6 @@ export interface RedisConnectionConfig {exitOnRepeatedErrors?: boolean}-/**- * Configuration needed to create Redis pool instances.- */export interface RedisPoolConfig {connection: RedisConnectionConfigpoolMinSize: number@@ -66,22 +58,30 @@ export function createRedisPoolFromConfig(config: RedisPoolConfig): RedisPool {}/**- * Scope entry for a `RedisPool`. `start` creates the pool (which connects- * to Redis eagerly via `autostart`), `stop` drains the pool then clears- * it so all connections are released.+ * Scope entry for a `RedisPool`. `start` creates the pool and resolves after+ * one connection answers. A pool with `exitOnRepeatedErrors: false` starts+ * without a connection, because an outage of an optional store must not block+ * startup. `stop` drains the pool then clears it so all connections are released.*/export class RedisPoolComponent {constructor(private readonly config: RedisPoolConfig) {}- start(): Promise<{ value: RedisPool; stop: () => Promise<void> }> {+ async start(): Promise<{ value: RedisPool; stop: () => Promise<void> }> {const pool = createRedisPoolFromConfig(this.config)- return Promise.resolve({- value: pool,- stop: async (): Promise<void> => {- await pool.drain()- await pool.clear()- },- })+ const stop = async (): Promise<void> => {+ await pool.drain()+ await pool.clear()+ }+ if (this.config.connection.exitOnRepeatedErrors === false) {+ return { value: pool, stop }+ }+ try {+ await pool.release(await pool.acquire())+ } catch (error) {+ await stop()+ throw error+ }+ return { value: pool, stop }}}