Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): add a shared idempotency key store for workflows (#120446)

Harley Alexander19c6ca4fa2d9
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 83adc533cf88..10c569db9867 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -3,6 +3,7 @@ import express from 'ultimate-express'
import { z } from 'zod'
import { ModifiedRequest } from '~/common/api/router'
+import type { RedisV2 } from '~/common/redis/redis-v2'
import { logger } from '~/common/utils/logger'
import { UUID, UUIDT, delay } from '~/common/utils/utils'
import { LogRecord } from '~/logs/log-record-avro'
@@ -136,7 +137,10 @@ function canDedupeByEmail(hogFlow: { actions?: unknown }): boolean {
}
export type CdpApiConfig = PluginsServerConfig
-export type CdpApiDeps = CdpConsumerBaseDeps
+export type CdpApiDeps = CdpConsumerBaseDeps & {
+ /** Holds caller-supplied idempotency keys. Null or unset makes the store report itself unavailable. */
+ idempotencyValkey?: RedisV2 | null
+}
// Null means the cohort asks to wait but its deadline is unreadable. Queuing it unparked would send to a
// cohort that is still calculating, so the caller rejects it.
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index 6f1eaff31f18..45a0652127c5 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -104,6 +104,13 @@ export type CdpConfig = ClickhouseConfig & {
CDP_FREQUENCY_CAP_VALKEY_PASSWORD: string
CDP_FREQUENCY_CAP_VALKEY_TLS: boolean
+ // Valkey for caller-supplied idempotency keys, such as an `Idempotency-Key` header. When it is
+ // unset, each route decides what a request with a key does.
+ CDP_IDEMPOTENCY_VALKEY_HOST: string
+ CDP_IDEMPOTENCY_VALKEY_PORT: number
+ CDP_IDEMPOTENCY_VALKEY_PASSWORD: string
+ CDP_IDEMPOTENCY_VALKEY_TLS: boolean
+
CDP_SES_RATE_LIMIT_REFILL_PER_SECOND: number
CDP_SES_RATE_LIMIT_CAPACITY: number
CDP_SES_RATE_LIMIT_THROTTLED_POLL_DELAY_MS: number
@@ -334,6 +341,11 @@ export function getDefaultCdpConfig(): CdpConfig {
CDP_FREQUENCY_CAP_VALKEY_PASSWORD: '',
CDP_FREQUENCY_CAP_VALKEY_TLS: false,
+ CDP_IDEMPOTENCY_VALKEY_HOST: isTestEnv() || isDevEnv() ? '127.0.0.1' : '',
+ CDP_IDEMPOTENCY_VALKEY_PORT: isTestEnv() || isDevEnv() ? 6390 : 6379,
+ CDP_IDEMPOTENCY_VALKEY_PASSWORD: '',
+ CDP_IDEMPOTENCY_VALKEY_TLS: false,
+
CDP_SES_RATE_LIMIT_REFILL_PER_SECOND: 100,
CDP_SES_RATE_LIMIT_CAPACITY: 50,
CDP_SES_RATE_LIMIT_THROTTLED_POLL_DELAY_MS: 250,
diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts
new file mode 100644
index 000000000000..0b9b90b30946
--- /dev/null
+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts
@@ -0,0 +1,132 @@
+import Redis from 'ioredis'
+import { register } from 'prom-client'
+
+import { defaultConfig } from '~/common/config/config'
+import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
+import * as redisDb from '~/common/utils/db/redis'
+import { createRedisClient } from '~/common/utils/db/redis'
+import { logger } from '~/common/utils/logger'
+import { UUIDT } from '~/common/utils/utils'
+
+import { IdempotencyKeys, createIdempotencyValkeyPool } from './idempotency-keys'
+
+describe('IdempotencyKeys', () => {
+ // Alerts read this counter, so an outcome that stops being counted silences them.
+ const operations = async (scope: string, operation: string, outcome: string): Promise<number> =>
+ (await register.getSingleMetric('cdp_hogflow_idempotency_operations_total')!.get()).values.find(
+ (v) => v.labels.scope === scope && v.labels.operation === operation && v.labels.outcome === outcome
+ )?.value ?? 0
+
+ let valkey: RedisV2
+ let keys: IdempotencyKeys
+ let key: string
+
+ beforeAll(() => {
+ valkey = createIdempotencyValkeyPool(defaultConfig)!
+ keys = new IdempotencyKeys(valkey, 'test')
+ })
+
+ beforeEach(() => {
+ // A fresh key per test, so no test reads a key another one left in Valkey.
+ key = `key-${new UUIDT().toString()}`
+ })
+
+ it('holds a key while the first request queues, then replays its run', async () => {
+ const replayedBefore = await operations('test', 'reserve', 'replayed')
+ expect(await keys.reserve(1, key, 'request-a', 'run-1')).toEqual({ status: 'reserved' })
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'in_flight' })
+ expect(await keys.reserve(1, key, 'request-b', 'run-3')).toEqual({ status: 'conflict' })
+
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+
+ expect(await keys.reserve(1, key, 'request-a', 'run-4')).toEqual({ status: 'replayed', invocationId: 'run-1' })
+ expect(await keys.reserve(2, key, 'request-a', 'run-5')).toEqual({ status: 'reserved' })
+ expect(await operations('test', 'reserve', 'replayed')).toEqual(replayedBefore + 1)
+ })
+
+ it('frees a key only for the request that holds it', async () => {
+ await keys.reserve(1, key, 'request-a', 'run-1')
+
+ await keys.release(1, key, 'run-2')
+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({ status: 'in_flight' })
+
+ await keys.release(1, key, 'run-1')
+ expect(await keys.reserve(1, key, 'request-a', 'run-4')).toEqual({ status: 'reserved' })
+ })
+
+ it('records the run when the queue write outlasts the pending hold', async () => {
+ await keys.reserve(1, key, 'request-a', 'run-1')
+ // The pending hold expires while the queue write is still running.
+ await valkey.useClient({ name: 'test-expire' }, (client) => client.del(keys['redisKey'](1, key)))
+
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'replayed', invocationId: 'run-1' })
+ })
+
+ it('reports the store as unavailable when it is not configured', async () => {
+ const unavailableBefore = await operations('unconfigured', 'reserve', 'unavailable')
+
+ expect(await new IdempotencyKeys(null, 'unconfigured').reserve(1, key, 'request-a', 'run-1')).toEqual({
+ status: 'unavailable',
+ })
+ expect(await operations('unconfigured', 'reserve', 'unavailable')).toEqual(unavailableBefore + 1)
+ })
+
+ it.each([
+ [
+ 'reordered object keys',
+ { a: 1, b: { c: 2, d: [{ e: 3, f: 4 }] } },
+ { b: { d: [{ f: 4, e: 3 }], c: 2 }, a: 1 },
+ true,
+ ],
+ ['a changed value', { a: 1 }, { a: 2 }, false],
+ ['reordered array items', [1, 2], [2, 1], false],
+ ])('fingerprints requests that differ by %s', (_desc, first, second, same) => {
+ expect(IdempotencyKeys.fingerprint(first) === IdempotencyKeys.fingerprint(second)).toBe(same)
+ })
+
+ it('pauses between failed connections and reports no delay after the acquire timeout', async () => {
+ jest.useFakeTimers()
+ const create = jest.spyOn(redisDb, 'createRedisFromConfig').mockRejectedValue(new Error('WRONGPASS'))
+ const warn = jest.spyOn(logger, 'warn')
+ try {
+ const pool = createRedisV2PoolFromConfig({
+ connection: { url: '127.0.0.1', name: 'pause-test', exitOnRepeatedErrors: false },
+ poolMinSize: 0,
+ poolMaxSize: 1,
+ acquireTimeoutMillis: 1500,
+ })
+ const call = expect(
+ pool.useClient({ name: 'test', timeout: 2000 }, (client) => client.ping())
+ ).rejects.toThrow()
+
+ await jest.advanceTimersByTimeAsync(1500)
+ await call
+ // Without the pause, generic-pool retries the failed create thousands of times in this window.
+ // Only this pool's attempts count, because the suite's shared pool also creates connections.
+ const attempts = create.mock.calls.filter(([connection]) => connection.name === 'pause-test')
+ expect(attempts.length).toBeLessThanOrEqual(2)
+
+ await jest.advanceTimersByTimeAsync(2000)
+ expect(warn).not.toHaveBeenCalledWith('⌛', expect.stringContaining('delayed'), undefined)
+ } finally {
+ create.mockRestore()
+ warn.mockRestore()
+ jest.useRealTimers()
+ }
+ })
+
+ it('closes the client of an unreachable store instead of reconnecting until the process stops', async () => {
+ const disconnect = jest.spyOn(Redis.prototype, 'disconnect')
+ try {
+ // Nothing listens on port 1. A client left open keeps reconnecting and counts each error.
+ await expect(
+ createRedisClient('127.0.0.1', { port: 1, commandTimeout: 200 }, 'test', false)
+ ).rejects.toThrow()
+ expect(disconnect).toHaveBeenCalled()
+ } finally {
+ disconnect.mockRestore()
+ }
+ })
+})
diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts
new file mode 100644
index 000000000000..96d81a786cc3
--- /dev/null
+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts
@@ -0,0 +1,264 @@
+import { createHash } from 'crypto'
+import { Counter, Histogram } from 'prom-client'
+
+import type { CommonConfig } from '~/common/config'
+import { RedisClient, RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
+import { parseJSON } from '~/common/utils/json-parse'
+import { logger } from '~/common/utils/logger'
+
+import type { CdpConfig } from '../../config'
+
+// Covers a caller's timeout and retry. After this a pending key expires, so a crash between the
+// reservation and the queue write can cost a duplicate, but never a send the caller thinks happened.
+const PENDING_TTL_SECONDS = 60
+const QUEUED_TTL_SECONDS = 24 * 60 * 60
+// A hung Valkey must not hold the caller's request open. Each route decides what an unavailable store means.
+const VALKEY_TIMEOUT_MS = 1000
+// Deletes the key only while it still holds this request's reservation, so a late release never frees
+// a key that another request reserved after this one expired.
+const RELEASE_SCRIPT = `local v = redis.call('GET', KEYS[1]) if v and cjson.decode(v).invocationId == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0`
+// Records the queued run when the key is absent or still holds this request's reservation. The key is
+// absent when the queue write took longer than the pending TTL, and the run is queued all the same.
+const MARK_QUEUED_SCRIPT = `local v = redis.call('GET', KEYS[1]) if (not v) or cjson.decode(v).invocationId == ARGV[2] then redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[3]) return 1 end return 0`
+const MARK_QUEUED_ATTEMPTS = 2
+
+const idempotencyOperationsTotal = new Counter({
+ name: 'cdp_hogflow_idempotency_operations_total',
+ help: 'Idempotency key store operations by route scope, operation and outcome. A reserve outcome of unavailable means the request ran without deduplication or was refused, by route.',
+ labelNames: ['scope', 'operation', 'outcome'],
+})
+
+const idempotencyValkeyDurationSeconds = new Histogram({
+ name: 'cdp_hogflow_idempotency_valkey_duration_seconds',
+ help: 'Duration of idempotency key store calls to Valkey, by route scope and operation. Calls stop at the 1 second timeout.',
+ labelNames: ['scope', 'operation'],
+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+})
+
+type IdempotencyOperation = 'reserve' | 'mark_queued' | 'release'
+
+const withTimeout = async <T>(promise: Promise<T>, timeoutMs: number): Promise<T> => {
+ let timeoutId: NodeJS.Timeout | undefined
+ try {
+ return await Promise.race([
+ promise,
+ new Promise<never>((_, reject) => {
+ timeoutId = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs)
+ }),
+ ])
+ } finally {
+ clearTimeout(timeoutId)
+ }
+}
+
+export type IdempotencyValkeyConfig = Pick<
+ CdpConfig,
+ | 'CDP_IDEMPOTENCY_VALKEY_HOST'
+ | 'CDP_IDEMPOTENCY_VALKEY_PORT'
+ | 'CDP_IDEMPOTENCY_VALKEY_PASSWORD'
+ | 'CDP_IDEMPOTENCY_VALKEY_TLS'
+> &
+ Pick<CommonConfig, 'REDIS_POOL_MIN_SIZE' | 'REDIS_POOL_MAX_SIZE'>
+
+/**
+ * Creates a connection to the Valkey that holds caller-supplied idempotency keys. Returns null when
+ * the host is unset, and the store then reports itself unavailable.
+ */
+export function createIdempotencyValkeyPool(config: IdempotencyValkeyConfig): RedisV2 | null {
+ const name = 'workflows-idempotency'
+ if (!config.CDP_IDEMPOTENCY_VALKEY_HOST) {
+ logger.info('🔑', `[${name}] no idempotency Valkey host configured, so the idempotency store is unavailable`)
+ return null
+ }
+
+ return createRedisV2PoolFromConfig({
+ connection: {
+ url: config.CDP_IDEMPOTENCY_VALKEY_HOST,
+ options: {
+ port: config.CDP_IDEMPOTENCY_VALKEY_PORT,
+ password: config.CDP_IDEMPOTENCY_VALKEY_PASSWORD,
+ tls: config.CDP_IDEMPOTENCY_VALKEY_TLS ? {} : undefined,
+ // The caller's timeout only stops the wait. This one aborts the command, so a hung
+ // Valkey cannot keep every pool connection borrowed.
+ commandTimeout: VALKEY_TIMEOUT_MS,
+ },
+ name,
+ // The store is optional, so its outage must not stop cdp-api.
+ exitOnRepeatedErrors: false,
+ },
+ // No connection at boot, so cdp-api starts without the store.
+ poolMinSize: 0,
+ poolMaxSize: config.REDIS_POOL_MAX_SIZE,
+ acquireTimeoutMillis: VALKEY_TIMEOUT_MS,
+ })
+}
+
+const sortKeys = (value: unknown): unknown => {
+ if (Array.isArray(value)) {
+ return value.map(sortKeys)
+ }
+ if (value !== null && typeof value === 'object') {
+ return Object.fromEntries(
+ Object.keys(value)
+ .sort()
+ .map((key) => [key, sortKeys((value as Record<string, unknown>)[key])])
+ )
+ }
+ return value
+}
+
+export type IdempotencyReservation =
+ | { status: 'reserved' }
+ // The key already queued a run for the same request.
+ | { status: 'replayed'; invocationId: string }
+ // The first request with the key has not finished queueing yet.
+ | { status: 'in_flight' }
+ // The key was used for a different request.
+ | { status: 'conflict' }
+ | { status: 'unavailable' }
+
+type StoredKey = { fingerprint: string; invocationId: string; queued: boolean }
+
+/**
+ * Caller-supplied idempotency keys, scoped to a team. A key reserves a run before it is queued, and
+ * holds the run's id for 24 hours once it is. A queue failure releases the key, so the caller's
+ * retry can queue the run.
+ */
+export class IdempotencyKeys {
+ constructor(
+ private valkey: RedisV2 | null,
+ private scope: string
+ ) {}
+
+ static fingerprint(request: unknown): string {
+ // Callers may rebuild the same request with its object keys in another order on a retry.
+ return createHash('sha256')
+ .update(JSON.stringify(sortKeys(request)))
+ .digest('hex')
+ }
+
+ async reserve(
+ teamId: number,
+ key: string,
+ fingerprint: string,
+ invocationId: string
+ ): Promise<IdempotencyReservation> {
+ const reservation = await this.reserveInStore(teamId, key, fingerprint, invocationId)
+ this.count('reserve', reservation.status)
+ return reservation
+ }
+
+ private async reserveInStore(
+ teamId: number,
+ key: string,
+ fingerprint: string,
+ invocationId: string
+ ): Promise<IdempotencyReservation> {
+ const valkey = this.valkey
+ if (!valkey) {
+ return { status: 'unavailable' }
+ }
+ const value: StoredKey = { fingerprint, invocationId, queued: false }
+ let existing: string | null
+ try {
+ // SET ... NX GET sets the key when absent and returns the existing value when present.
+ // ioredis 4.x types only describe 'OK' | null, so cast to the GET payload.
+ existing = await this.timed('reserve', () =>
+ valkey.useClient(
+ { name: 'idempotency-reserve' },
+ (client: RedisClient) =>
+ client.set(this.redisKey(teamId, key), JSON.stringify(value), [
+ 'EX',
+ String(PENDING_TTL_SECONDS),
+ 'NX',
+ 'GET',
+ ]) as Promise<string | null>
+ )
+ )
+ } catch (error) {
+ logger.error('🔑', 'idempotency key reservation failed', { scope: this.scope, error: String(error) })
+ return { status: 'unavailable' }
+ }
+ if (existing === null) {
+ return { status: 'reserved' }
+ }
+ const stored = parseJSON(existing) as StoredKey
+ if (stored.fingerprint !== fingerprint) {
+ return { status: 'conflict' }
+ }
+ return stored.queued ? { status: 'replayed', invocationId: stored.invocationId } : { status: 'in_flight' }
+ }
+
+ async markQueued(teamId: number, key: string, fingerprint: string, invocationId: string): Promise<void> {
+ const value: StoredKey = { fingerprint, invocationId, queued: true }
+ for (let attempt = 1; attempt <= MARK_QUEUED_ATTEMPTS; attempt++) {
+ try {
+ const written = await this.timed(
+ 'mark_queued',
+ () =>
+ this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>
+ client.eval(
+ MARK_QUEUED_SCRIPT,
+ 1,
+ this.redisKey(teamId, key),
+ JSON.stringify(value),
+ invocationId,
+ QUEUED_TTL_SECONDS
+ )
+ ) ?? Promise.resolve(null)
+ )
+ if (written === 0) {
+ logger.warn('🔑', 'idempotency key held by another request, run not recorded', {
+ scope: this.scope,
+ })
+ }
+ this.count('mark_queued', written === 0 ? 'not_owned' : 'written')
+ return
+ } catch (error) {
+ // The run is already queued, so the request still succeeds. Without this record a retry
+ // after the pending TTL queues the run again.
+ logger.error('🔑', 'idempotency key completion failed', {
+ scope: this.scope,
+ attempt,
+ error: String(error),
+ })
+ }
+ }
+ this.count('mark_queued', 'failed')
+ }
+
+ async release(teamId: number, key: string, invocationId: string): Promise<void> {
+ try {
+ const released = await this.timed(
+ 'release',
+ () =>
+ this.valkey?.useClient({ name: 'idempotency-release' }, (client) =>
+ client.eval(RELEASE_SCRIPT, 1, this.redisKey(teamId, key), invocationId)
+ ) ?? Promise.resolve(null)
+ )
+ this.count('release', released === 1 ? 'released' : 'not_owned')
+ } catch (error) {
+ logger.error('🔑', 'idempotency key release failed', { scope: this.scope, error: String(error) })
+ this.count('release', 'failed')
+ }
+ }
+
+ private count(operation: IdempotencyOperation, outcome: string): void {
+ idempotencyOperationsTotal.inc({ scope: this.scope, operation, outcome })
+ }
+
+ private async timed<T>(operation: IdempotencyOperation, call: () => Promise<T>): Promise<T> {
+ const stopTimer = idempotencyValkeyDurationSeconds.startTimer({ scope: this.scope, operation })
+ try {
+ return await withTimeout(call(), VALKEY_TIMEOUT_MS)
+ } finally {
+ stopTimer()
+ }
+ }
+
+ private redisKey(teamId: number, key: string): string {
+ // Hashed so a long or unusual caller key can't shape the Valkey key.
+ const keyHash = createHash('sha256').update(key).digest('hex')
+ return `@posthog/idempotency/${this.scope}/${teamId}/${keyHash}`
+ }
+}
diff --git a/nodejs/src/common/redis/redis-v2.ts b/nodejs/src/common/redis/redis-v2.ts
index 4e4f7e91e4c0..35306883e611 100644
--- a/nodejs/src/common/redis/redis-v2.ts
+++ b/nodejs/src/common/redis/redis-v2.ts
@@ -35,11 +35,27 @@ export type RedisV2 = {
) => Promise<Array<[Error | null, any]> | null>
}
+// A non-fatal client can fail to connect at once, for example on a wrong password. generic-pool retries a
+// failed create immediately while a caller waits, so without this pause the pool reconnects in a tight loop.
+const NON_FATAL_CREATE_RETRY_DELAY_MS = 1000
+
export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 => {
+ const nonFatal = config.connection.exitOnRepeatedErrors === false
+ let lastCreateFailureAt = 0
const pool = createPool<RedisClient>(
{
create: async () => {
- const client = await createRedisFromConfig(config.connection)
+ const wait = lastCreateFailureAt + NON_FATAL_CREATE_RETRY_DELAY_MS - Date.now()
+ if (nonFatal && wait > 0) {
+ await new Promise((resolve) => setTimeout(resolve, wait))
+ }
+ let client: Awaited<ReturnType<typeof createRedisFromConfig>>
+ try {
+ client = await createRedisFromConfig(config.connection)
+ } catch (error) {
+ lastCreateFailureAt = Date.now()
+ throw error
+ }
defineLuaTokenBucketV2(client)
defineLuaTokenBucketV3(client)
@@ -54,6 +70,7 @@ export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 =>
min: config.poolMinSize,
max: config.poolMaxSize,
autostart: true,
+ ...(config.acquireTimeoutMillis !== undefined ? { acquireTimeoutMillis: config.acquireTimeoutMillis } : {}),
}
)
@@ -63,7 +80,14 @@ export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 =>
undefined,
options.timeout
)
- const client = await pool.acquire()
+ let client: RedisClient
+ try {
+ client = await pool.acquire()
+ } catch (error) {
+ // A pool with acquireTimeoutMillis rejects here, and the guard must not report a delay afterwards.
+ clearTimeout(timeout)
+ throw error
+ }
try {
return await callback(client)
diff --git a/nodejs/src/common/utils/db/redis.ts b/nodejs/src/common/utils/db/redis.ts
index cfe7b7ce6fd0..f6aa01137904 100644
--- a/nodejs/src/common/utils/db/redis.ts
+++ b/nodejs/src/common/utils/db/redis.ts
@@ -9,6 +9,7 @@ import { captureException } from '../posthog'
/** Number of Redis error events until the server is killed gracefully. */
const REDIS_ERROR_COUNTER_LIMIT = 10
+const NON_FATAL_CONNECT_TIMEOUT_MS = 10_000
/**
* Configuration for a Redis connection.
@@ -19,6 +20,12 @@ export interface RedisConnectionConfig {
url: string
options?: RedisOptions
name?: string
+ /**
+ * Stops the process after repeated connection errors. Defaults to true. Set it to false for an
+ * optional store, so an outage of that store does not take down every route of the service.
+ * A client that cannot connect is then closed, and its caller sees the error.
+ */
+ exitOnRepeatedErrors?: boolean
}
/**
@@ -38,7 +45,7 @@ export interface RedisPoolConfig {
}
export async function createRedisFromConfig(config: RedisConnectionConfig): Promise<Redis.Redis> {
- return createRedisClient(config.url, config.options, config.name)
+ return createRedisClient(config.url, config.options, config.name, config.exitOnRepeatedErrors ?? true)
}
export function createRedisPoolFromConfig(config: RedisPoolConfig): RedisPool {
@@ -99,7 +106,8 @@ export function getRedisHost(url: string, options?: RedisOptions): string {
export async function createRedisClient(
url: string,
options?: RedisOptions,
- connectionName?: string
+ connectionName?: string,
+ exitOnRepeatedErrors: boolean = true
): Promise<Redis.Redis> {
const redis = new Redis(url, {
...options,
@@ -112,8 +120,11 @@ export async function createRedisClient(
redis
.on('error', (error) => {
errorCounter++
- captureException(error)
- if (errorCounter > REDIS_ERROR_COUNTER_LIMIT) {
+ // A non-fatal client reconnects on every error, so only its first error is reported.
+ if (exitOnRepeatedErrors || errorCounter === 1) {
+ captureException(error)
+ }
+ if (exitOnRepeatedErrors && errorCounter > REDIS_ERROR_COUNTER_LIMIT) {
logger.error(
'😡',
`${connectionId}Redis error encountered! host: ${redisHost} Enough of this, I quit!`,
@@ -133,6 +144,28 @@ export async function createRedisClient(
logger.info('✅', `${connectionId}Connected to Redis!`, redisHost)
}
})
- await redis.info()
+ try {
+ if (exitOnRepeatedErrors) {
+ await redis.info()
+ } else {
+ // A command waits offline while the client reconnects, so commandTimeout never starts for it.
+ let timer: NodeJS.Timeout | undefined
+ await Promise.race([
+ redis.info(),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error(`${connectionId}Redis did not connect`)),
+ options?.commandTimeout ?? NON_FATAL_CONNECT_TIMEOUT_MS
+ )
+ }),
+ ]).finally(() => clearTimeout(timer))
+ }
+ } catch (error) {
+ if (!exitOnRepeatedErrors) {
+ // Nothing holds this client, so it would otherwise reconnect forever.
+ redis.disconnect()
+ }
+ throw error
+ }
return redis
}
diff --git a/nodejs/src/server.ts b/nodejs/src/server.ts
index 3f5d995b513c..81391973e04b 100644
--- a/nodejs/src/server.ts
+++ b/nodejs/src/server.ts
@@ -45,6 +45,7 @@ import { CyclotronV2JanitorService, CyclotronV2Manager, CyclotronV2Worker } from
import { HogFlowScheduleService } from './cdp/services/hogflow-schedule/hogflow-schedule.service'
import { HOGFLOW_BATCH_RESOLVE_QUEUE } from './cdp/services/hogflows/batch-resolver.types'
import { HogFlowBatchPersonQueryService } from './cdp/services/hogflows/hogflow-batch-person-query.service'
+import { createIdempotencyValkeyPool } from './cdp/services/hogflows/idempotency-keys'
import { CyclotronJobQueueKafka } from './cdp/services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from './cdp/services/job-queue/job-queue-postgres-v2'
import { CyclotronJobQueueRateLimitedPostgresV2 } from './cdp/services/job-queue/job-queue-rate-limited-postgres-v2'
@@ -244,7 +245,7 @@ export class PluginServer implements NodeServer {
: null
const api = new CdpApi(
this.config,
- cdpDeps!,
+ { ...cdpDeps!, idempotencyValkey: createIdempotencyValkeyPool(this.config) },
{ hogQueue: kafkaQueue, hogflowQueue: postgresV2Queue },
batchResolverProducer
)