Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(ingestion): fail startup when a redis pool cannot connect (#122654)

Paweł Ledwoń1a2d4398b712
diff --git a/nodejs/src/common/utils/db/redis-pool-component.test.ts b/nodejs/src/common/utils/db/redis-pool-component.test.ts
new file mode 100644
index 000000000000..0dfa1cf26baf
--- /dev/null
+++ b/nodejs/src/common/utils/db/redis-pool-component.test.ts
@@ -0,0 +1,34 @@
+import { RedisConnectionConfig, RedisPoolComponent } from './redis'
+
+const giveUpBeforeErrorLimit = (attempt: number): number | null => (attempt < 5 ? 25 : null)
+
+const unreachableRedis = (overrides: Partial<RedisConnectionConfig>): RedisConnectionConfig => ({
+ url: '127.0.0.1',
+ options: { port: 1, commandTimeout: 100, retryStrategy: giveUpBeforeErrorLimit },
+ name: 'unreachable-test',
+ ...overrides,
+})
+
+describe('RedisPoolComponent', () => {
+ it('fails to start when Redis is unreachable', async () => {
+ const component = new RedisPoolComponent({
+ connection: unreachableRedis({}),
+ poolMinSize: 1,
+ poolMaxSize: 1,
+ acquireTimeoutMillis: 200,
+ })
+
+ await expect(component.start()).rejects.toThrow()
+ })
+
+ it('starts a non-fatal pool when Redis is unreachable', async () => {
+ const component = new RedisPoolComponent({
+ connection: unreachableRedis({ exitOnRepeatedErrors: false }),
+ poolMinSize: 1,
+ poolMaxSize: 1,
+ })
+
+ const started = await component.start()
+ await started.stop()
+ })
+})
diff --git a/nodejs/src/common/utils/db/redis.ts b/nodejs/src/common/utils/db/redis.ts
index f6aa01137904..839cd89cef7e 100644
--- a/nodejs/src/common/utils/db/redis.ts
+++ b/nodejs/src/common/utils/db/redis.ts
@@ -7,15 +7,10 @@ import { logger } from '../../utils/logger'
import { killGracefully } from '../../utils/utils'
import { captureException } from '../posthog'
-/** Number of Redis error events until the server is killed gracefully. */
+/** Redis errors a fatal client tolerates. The next error stops the process. */
const REDIS_ERROR_COUNTER_LIMIT = 10
const NON_FATAL_CONNECT_TIMEOUT_MS = 10_000
-/**
- * Configuration for a Redis connection.
- * Consumers should build this config inline where they create Redis connections,
- * rather than relying on centralized builder functions.
- */
export interface RedisConnectionConfig {
url: string
options?: RedisOptions
@@ -28,9 +23,6 @@ export interface RedisConnectionConfig {
exitOnRepeatedErrors?: boolean
}
-/**
- * Configuration needed to create Redis pool instances.
- */
export interface RedisPoolConfig {
connection: RedisConnectionConfig
poolMinSize: number
@@ -66,22 +58,30 @@ export function createRedisPoolFromConfig(config: RedisPoolConfig): RedisPool {
}
/**
- * Scope entry for a `RedisPool`. `start` creates the pool (which connects
- * to Redis eagerly via `autostart`), `stop` drains the pool then clears
- * it so all connections are released.
+ * Scope entry for a `RedisPool`. `start` creates the pool and resolves after
+ * one connection answers. A pool with `exitOnRepeatedErrors: false` starts
+ * without a connection, because an outage of an optional store must not block
+ * startup. `stop` drains the pool then clears it so all connections are released.
*/
export class RedisPoolComponent {
constructor(private readonly config: RedisPoolConfig) {}
- start(): Promise<{ value: RedisPool; stop: () => Promise<void> }> {
+ async start(): Promise<{ value: RedisPool; stop: () => Promise<void> }> {
const pool = createRedisPoolFromConfig(this.config)
- return Promise.resolve({
- value: pool,
- stop: async (): Promise<void> => {
- await pool.drain()
- await pool.clear()
- },
- })
+ const stop = async (): Promise<void> => {
+ await pool.drain()
+ await pool.clear()
+ }
+ if (this.config.connection.exitOnRepeatedErrors === false) {
+ return { value: pool, stop }
+ }
+ try {
+ await pool.release(await pool.acquire())
+ } catch (error) {
+ await stop()
+ throw error
+ }
+ return { value: pool, stop }
}
}