:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): add a shared idempotency key store for workflows (#120446)
Harley Alexander19c6ca4fa2d9
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 83adc533cf88..10c569db9867 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -3,6 +3,7 @@ import express from 'ultimate-express'import { z } from 'zod'import { ModifiedRequest } from '~/common/api/router'+import type { RedisV2 } from '~/common/redis/redis-v2'import { logger } from '~/common/utils/logger'import { UUID, UUIDT, delay } from '~/common/utils/utils'import { LogRecord } from '~/logs/log-record-avro'@@ -136,7 +137,10 @@ function canDedupeByEmail(hogFlow: { actions?: unknown }): boolean {}export type CdpApiConfig = PluginsServerConfig-export type CdpApiDeps = CdpConsumerBaseDeps+export type CdpApiDeps = CdpConsumerBaseDeps & {+ /** Holds caller-supplied idempotency keys. Null or unset makes the store report itself unavailable. */+ idempotencyValkey?: RedisV2 | null+}// Null means the cohort asks to wait but its deadline is unreadable. Queuing it unparked would send to a// cohort that is still calculating, so the caller rejects it.diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 6f1eaff31f18..45a0652127c5 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -104,6 +104,13 @@ export type CdpConfig = ClickhouseConfig & {CDP_FREQUENCY_CAP_VALKEY_PASSWORD: stringCDP_FREQUENCY_CAP_VALKEY_TLS: boolean+ // Valkey for caller-supplied idempotency keys, such as an `Idempotency-Key` header. When it is+ // unset, each route decides what a request with a key does.+ CDP_IDEMPOTENCY_VALKEY_HOST: string+ CDP_IDEMPOTENCY_VALKEY_PORT: number+ CDP_IDEMPOTENCY_VALKEY_PASSWORD: string+ CDP_IDEMPOTENCY_VALKEY_TLS: boolean+CDP_SES_RATE_LIMIT_REFILL_PER_SECOND: numberCDP_SES_RATE_LIMIT_CAPACITY: numberCDP_SES_RATE_LIMIT_THROTTLED_POLL_DELAY_MS: number@@ -334,6 +341,11 @@ export function getDefaultCdpConfig(): CdpConfig {CDP_FREQUENCY_CAP_VALKEY_PASSWORD: '',CDP_FREQUENCY_CAP_VALKEY_TLS: false,+ CDP_IDEMPOTENCY_VALKEY_HOST: isTestEnv() || isDevEnv() ? '127.0.0.1' : '',+ CDP_IDEMPOTENCY_VALKEY_PORT: isTestEnv() || isDevEnv() ? 6390 : 6379,+ CDP_IDEMPOTENCY_VALKEY_PASSWORD: '',+ CDP_IDEMPOTENCY_VALKEY_TLS: false,+CDP_SES_RATE_LIMIT_REFILL_PER_SECOND: 100,CDP_SES_RATE_LIMIT_CAPACITY: 50,CDP_SES_RATE_LIMIT_THROTTLED_POLL_DELAY_MS: 250,diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.tsnew file mode 100644index 000000000000..0b9b90b30946--- /dev/null+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts@@ -0,0 +1,132 @@+import Redis from 'ioredis'+import { register } from 'prom-client'++import { defaultConfig } from '~/common/config/config'+import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'+import * as redisDb from '~/common/utils/db/redis'+import { createRedisClient } from '~/common/utils/db/redis'+import { logger } from '~/common/utils/logger'+import { UUIDT } from '~/common/utils/utils'++import { IdempotencyKeys, createIdempotencyValkeyPool } from './idempotency-keys'++describe('IdempotencyKeys', () => {+ // Alerts read this counter, so an outcome that stops being counted silences them.+ const operations = async (scope: string, operation: string, outcome: string): Promise<number> =>+ (await register.getSingleMetric('cdp_hogflow_idempotency_operations_total')!.get()).values.find(+ (v) => v.labels.scope === scope && v.labels.operation === operation && v.labels.outcome === outcome+ )?.value ?? 0++ let valkey: RedisV2+ let keys: IdempotencyKeys+ let key: string++ beforeAll(() => {+ valkey = createIdempotencyValkeyPool(defaultConfig)!+ keys = new IdempotencyKeys(valkey, 'test')+ })++ beforeEach(() => {+ // A fresh key per test, so no test reads a key another one left in Valkey.+ key = `key-${new UUIDT().toString()}`+ })++ it('holds a key while the first request queues, then replays its run', async () => {+ const replayedBefore = await operations('test', 'reserve', 'replayed')+ expect(await keys.reserve(1, key, 'request-a', 'run-1')).toEqual({ status: 'reserved' })+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'in_flight' })+ expect(await keys.reserve(1, key, 'request-b', 'run-3')).toEqual({ status: 'conflict' })++ await keys.markQueued(1, key, 'request-a', 'run-1')++ expect(await keys.reserve(1, key, 'request-a', 'run-4')).toEqual({ status: 'replayed', invocationId: 'run-1' })+ expect(await keys.reserve(2, key, 'request-a', 'run-5')).toEqual({ status: 'reserved' })+ expect(await operations('test', 'reserve', 'replayed')).toEqual(replayedBefore + 1)+ })++ it('frees a key only for the request that holds it', async () => {+ await keys.reserve(1, key, 'request-a', 'run-1')++ await keys.release(1, key, 'run-2')+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({ status: 'in_flight' })++ await keys.release(1, key, 'run-1')+ expect(await keys.reserve(1, key, 'request-a', 'run-4')).toEqual({ status: 'reserved' })+ })++ it('records the run when the queue write outlasts the pending hold', async () => {+ await keys.reserve(1, key, 'request-a', 'run-1')+ // The pending hold expires while the queue write is still running.+ await valkey.useClient({ name: 'test-expire' }, (client) => client.del(keys['redisKey'](1, key)))++ await keys.markQueued(1, key, 'request-a', 'run-1')++ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'replayed', invocationId: 'run-1' })+ })++ it('reports the store as unavailable when it is not configured', async () => {+ const unavailableBefore = await operations('unconfigured', 'reserve', 'unavailable')++ expect(await new IdempotencyKeys(null, 'unconfigured').reserve(1, key, 'request-a', 'run-1')).toEqual({+ status: 'unavailable',+ })+ expect(await operations('unconfigured', 'reserve', 'unavailable')).toEqual(unavailableBefore + 1)+ })++ it.each([+ [+ 'reordered object keys',+ { a: 1, b: { c: 2, d: [{ e: 3, f: 4 }] } },+ { b: { d: [{ f: 4, e: 3 }], c: 2 }, a: 1 },+ true,+ ],+ ['a changed value', { a: 1 }, { a: 2 }, false],+ ['reordered array items', [1, 2], [2, 1], false],+ ])('fingerprints requests that differ by %s', (_desc, first, second, same) => {+ expect(IdempotencyKeys.fingerprint(first) === IdempotencyKeys.fingerprint(second)).toBe(same)+ })++ it('pauses between failed connections and reports no delay after the acquire timeout', async () => {+ jest.useFakeTimers()+ const create = jest.spyOn(redisDb, 'createRedisFromConfig').mockRejectedValue(new Error('WRONGPASS'))+ const warn = jest.spyOn(logger, 'warn')+ try {+ const pool = createRedisV2PoolFromConfig({+ connection: { url: '127.0.0.1', name: 'pause-test', exitOnRepeatedErrors: false },+ poolMinSize: 0,+ poolMaxSize: 1,+ acquireTimeoutMillis: 1500,+ })+ const call = expect(+ pool.useClient({ name: 'test', timeout: 2000 }, (client) => client.ping())+ ).rejects.toThrow()++ await jest.advanceTimersByTimeAsync(1500)+ await call+ // Without the pause, generic-pool retries the failed create thousands of times in this window.+ // Only this pool's attempts count, because the suite's shared pool also creates connections.+ const attempts = create.mock.calls.filter(([connection]) => connection.name === 'pause-test')+ expect(attempts.length).toBeLessThanOrEqual(2)++ await jest.advanceTimersByTimeAsync(2000)+ expect(warn).not.toHaveBeenCalledWith('⌛', expect.stringContaining('delayed'), undefined)+ } finally {+ create.mockRestore()+ warn.mockRestore()+ jest.useRealTimers()+ }+ })++ it('closes the client of an unreachable store instead of reconnecting until the process stops', async () => {+ const disconnect = jest.spyOn(Redis.prototype, 'disconnect')+ try {+ // Nothing listens on port 1. A client left open keeps reconnecting and counts each error.+ await expect(+ createRedisClient('127.0.0.1', { port: 1, commandTimeout: 200 }, 'test', false)+ ).rejects.toThrow()+ expect(disconnect).toHaveBeenCalled()+ } finally {+ disconnect.mockRestore()+ }+ })+})diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.tsnew file mode 100644index 000000000000..96d81a786cc3--- /dev/null+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts@@ -0,0 +1,264 @@+import { createHash } from 'crypto'+import { Counter, Histogram } from 'prom-client'++import type { CommonConfig } from '~/common/config'+import { RedisClient, RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'+import { parseJSON } from '~/common/utils/json-parse'+import { logger } from '~/common/utils/logger'++import type { CdpConfig } from '../../config'++// Covers a caller's timeout and retry. After this a pending key expires, so a crash between the+// reservation and the queue write can cost a duplicate, but never a send the caller thinks happened.+const PENDING_TTL_SECONDS = 60+const QUEUED_TTL_SECONDS = 24 * 60 * 60+// A hung Valkey must not hold the caller's request open. Each route decides what an unavailable store means.+const VALKEY_TIMEOUT_MS = 1000+// Deletes the key only while it still holds this request's reservation, so a late release never frees+// a key that another request reserved after this one expired.+const RELEASE_SCRIPT = `local v = redis.call('GET', KEYS[1]) if v and cjson.decode(v).invocationId == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0`+// Records the queued run when the key is absent or still holds this request's reservation. The key is+// absent when the queue write took longer than the pending TTL, and the run is queued all the same.+const MARK_QUEUED_SCRIPT = `local v = redis.call('GET', KEYS[1]) if (not v) or cjson.decode(v).invocationId == ARGV[2] then redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[3]) return 1 end return 0`+const MARK_QUEUED_ATTEMPTS = 2++const idempotencyOperationsTotal = new Counter({+ name: 'cdp_hogflow_idempotency_operations_total',+ help: 'Idempotency key store operations by route scope, operation and outcome. A reserve outcome of unavailable means the request ran without deduplication or was refused, by route.',+ labelNames: ['scope', 'operation', 'outcome'],+})++const idempotencyValkeyDurationSeconds = new Histogram({+ name: 'cdp_hogflow_idempotency_valkey_duration_seconds',+ help: 'Duration of idempotency key store calls to Valkey, by route scope and operation. Calls stop at the 1 second timeout.',+ labelNames: ['scope', 'operation'],+ buckets: [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],+})++type IdempotencyOperation = 'reserve' | 'mark_queued' | 'release'++const withTimeout = async <T>(promise: Promise<T>, timeoutMs: number): Promise<T> => {+ let timeoutId: NodeJS.Timeout | undefined+ try {+ return await Promise.race([+ promise,+ new Promise<never>((_, reject) => {+ timeoutId = setTimeout(() => reject(new Error(`timed out after ${timeoutMs}ms`)), timeoutMs)+ }),+ ])+ } finally {+ clearTimeout(timeoutId)+ }+}++export type IdempotencyValkeyConfig = Pick<+ CdpConfig,+ | 'CDP_IDEMPOTENCY_VALKEY_HOST'+ | 'CDP_IDEMPOTENCY_VALKEY_PORT'+ | 'CDP_IDEMPOTENCY_VALKEY_PASSWORD'+ | 'CDP_IDEMPOTENCY_VALKEY_TLS'+> &+ Pick<CommonConfig, 'REDIS_POOL_MIN_SIZE' | 'REDIS_POOL_MAX_SIZE'>++/**+ * Creates a connection to the Valkey that holds caller-supplied idempotency keys. Returns null when+ * the host is unset, and the store then reports itself unavailable.+ */+export function createIdempotencyValkeyPool(config: IdempotencyValkeyConfig): RedisV2 | null {+ const name = 'workflows-idempotency'+ if (!config.CDP_IDEMPOTENCY_VALKEY_HOST) {+ logger.info('🔑', `[${name}] no idempotency Valkey host configured, so the idempotency store is unavailable`)+ return null+ }++ return createRedisV2PoolFromConfig({+ connection: {+ url: config.CDP_IDEMPOTENCY_VALKEY_HOST,+ options: {+ port: config.CDP_IDEMPOTENCY_VALKEY_PORT,+ password: config.CDP_IDEMPOTENCY_VALKEY_PASSWORD,+ tls: config.CDP_IDEMPOTENCY_VALKEY_TLS ? {} : undefined,+ // The caller's timeout only stops the wait. This one aborts the command, so a hung+ // Valkey cannot keep every pool connection borrowed.+ commandTimeout: VALKEY_TIMEOUT_MS,+ },+ name,+ // The store is optional, so its outage must not stop cdp-api.+ exitOnRepeatedErrors: false,+ },+ // No connection at boot, so cdp-api starts without the store.+ poolMinSize: 0,+ poolMaxSize: config.REDIS_POOL_MAX_SIZE,+ acquireTimeoutMillis: VALKEY_TIMEOUT_MS,+ })+}++const sortKeys = (value: unknown): unknown => {+ if (Array.isArray(value)) {+ return value.map(sortKeys)+ }+ if (value !== null && typeof value === 'object') {+ return Object.fromEntries(+ Object.keys(value)+ .sort()+ .map((key) => [key, sortKeys((value as Record<string, unknown>)[key])])+ )+ }+ return value+}++export type IdempotencyReservation =+ | { status: 'reserved' }+ // The key already queued a run for the same request.+ | { status: 'replayed'; invocationId: string }+ // The first request with the key has not finished queueing yet.+ | { status: 'in_flight' }+ // The key was used for a different request.+ | { status: 'conflict' }+ | { status: 'unavailable' }++type StoredKey = { fingerprint: string; invocationId: string; queued: boolean }++/**+ * Caller-supplied idempotency keys, scoped to a team. A key reserves a run before it is queued, and+ * holds the run's id for 24 hours once it is. A queue failure releases the key, so the caller's+ * retry can queue the run.+ */+export class IdempotencyKeys {+ constructor(+ private valkey: RedisV2 | null,+ private scope: string+ ) {}++ static fingerprint(request: unknown): string {+ // Callers may rebuild the same request with its object keys in another order on a retry.+ return createHash('sha256')+ .update(JSON.stringify(sortKeys(request)))+ .digest('hex')+ }++ async reserve(+ teamId: number,+ key: string,+ fingerprint: string,+ invocationId: string+ ): Promise<IdempotencyReservation> {+ const reservation = await this.reserveInStore(teamId, key, fingerprint, invocationId)+ this.count('reserve', reservation.status)+ return reservation+ }++ private async reserveInStore(+ teamId: number,+ key: string,+ fingerprint: string,+ invocationId: string+ ): Promise<IdempotencyReservation> {+ const valkey = this.valkey+ if (!valkey) {+ return { status: 'unavailable' }+ }+ const value: StoredKey = { fingerprint, invocationId, queued: false }+ let existing: string | null+ try {+ // SET ... NX GET sets the key when absent and returns the existing value when present.+ // ioredis 4.x types only describe 'OK' | null, so cast to the GET payload.+ existing = await this.timed('reserve', () =>+ valkey.useClient(+ { name: 'idempotency-reserve' },+ (client: RedisClient) =>+ client.set(this.redisKey(teamId, key), JSON.stringify(value), [+ 'EX',+ String(PENDING_TTL_SECONDS),+ 'NX',+ 'GET',+ ]) as Promise<string | null>+ )+ )+ } catch (error) {+ logger.error('🔑', 'idempotency key reservation failed', { scope: this.scope, error: String(error) })+ return { status: 'unavailable' }+ }+ if (existing === null) {+ return { status: 'reserved' }+ }+ const stored = parseJSON(existing) as StoredKey+ if (stored.fingerprint !== fingerprint) {+ return { status: 'conflict' }+ }+ return stored.queued ? { status: 'replayed', invocationId: stored.invocationId } : { status: 'in_flight' }+ }++ async markQueued(teamId: number, key: string, fingerprint: string, invocationId: string): Promise<void> {+ const value: StoredKey = { fingerprint, invocationId, queued: true }+ for (let attempt = 1; attempt <= MARK_QUEUED_ATTEMPTS; attempt++) {+ try {+ const written = await this.timed(+ 'mark_queued',+ () =>+ this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>+ client.eval(+ MARK_QUEUED_SCRIPT,+ 1,+ this.redisKey(teamId, key),+ JSON.stringify(value),+ invocationId,+ QUEUED_TTL_SECONDS+ )+ ) ?? Promise.resolve(null)+ )+ if (written === 0) {+ logger.warn('🔑', 'idempotency key held by another request, run not recorded', {+ scope: this.scope,+ })+ }+ this.count('mark_queued', written === 0 ? 'not_owned' : 'written')+ return+ } catch (error) {+ // The run is already queued, so the request still succeeds. Without this record a retry+ // after the pending TTL queues the run again.+ logger.error('🔑', 'idempotency key completion failed', {+ scope: this.scope,+ attempt,+ error: String(error),+ })+ }+ }+ this.count('mark_queued', 'failed')+ }++ async release(teamId: number, key: string, invocationId: string): Promise<void> {+ try {+ const released = await this.timed(+ 'release',+ () =>+ this.valkey?.useClient({ name: 'idempotency-release' }, (client) =>+ client.eval(RELEASE_SCRIPT, 1, this.redisKey(teamId, key), invocationId)+ ) ?? Promise.resolve(null)+ )+ this.count('release', released === 1 ? 'released' : 'not_owned')+ } catch (error) {+ logger.error('🔑', 'idempotency key release failed', { scope: this.scope, error: String(error) })+ this.count('release', 'failed')+ }+ }++ private count(operation: IdempotencyOperation, outcome: string): void {+ idempotencyOperationsTotal.inc({ scope: this.scope, operation, outcome })+ }++ private async timed<T>(operation: IdempotencyOperation, call: () => Promise<T>): Promise<T> {+ const stopTimer = idempotencyValkeyDurationSeconds.startTimer({ scope: this.scope, operation })+ try {+ return await withTimeout(call(), VALKEY_TIMEOUT_MS)+ } finally {+ stopTimer()+ }+ }++ private redisKey(teamId: number, key: string): string {+ // Hashed so a long or unusual caller key can't shape the Valkey key.+ const keyHash = createHash('sha256').update(key).digest('hex')+ return `@posthog/idempotency/${this.scope}/${teamId}/${keyHash}`+ }+}diff --git a/nodejs/src/common/redis/redis-v2.ts b/nodejs/src/common/redis/redis-v2.tsindex 4e4f7e91e4c0..35306883e611 100644--- a/nodejs/src/common/redis/redis-v2.ts+++ b/nodejs/src/common/redis/redis-v2.ts@@ -35,11 +35,27 @@ export type RedisV2 = {) => Promise<Array<[Error | null, any]> | null>}+// A non-fatal client can fail to connect at once, for example on a wrong password. generic-pool retries a+// failed create immediately while a caller waits, so without this pause the pool reconnects in a tight loop.+const NON_FATAL_CREATE_RETRY_DELAY_MS = 1000+export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 => {+ const nonFatal = config.connection.exitOnRepeatedErrors === false+ let lastCreateFailureAt = 0const pool = createPool<RedisClient>({create: async () => {- const client = await createRedisFromConfig(config.connection)+ const wait = lastCreateFailureAt + NON_FATAL_CREATE_RETRY_DELAY_MS - Date.now()+ if (nonFatal && wait > 0) {+ await new Promise((resolve) => setTimeout(resolve, wait))+ }+ let client: Awaited<ReturnType<typeof createRedisFromConfig>>+ try {+ client = await createRedisFromConfig(config.connection)+ } catch (error) {+ lastCreateFailureAt = Date.now()+ throw error+ }defineLuaTokenBucketV2(client)defineLuaTokenBucketV3(client)@@ -54,6 +70,7 @@ export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 =>min: config.poolMinSize,max: config.poolMaxSize,autostart: true,+ ...(config.acquireTimeoutMillis !== undefined ? { acquireTimeoutMillis: config.acquireTimeoutMillis } : {}),})@@ -63,7 +80,14 @@ export const createRedisV2PoolFromConfig = (config: RedisPoolConfig): RedisV2 =>undefined,options.timeout)- const client = await pool.acquire()+ let client: RedisClient+ try {+ client = await pool.acquire()+ } catch (error) {+ // A pool with acquireTimeoutMillis rejects here, and the guard must not report a delay afterwards.+ clearTimeout(timeout)+ throw error+ }try {return await callback(client)diff --git a/nodejs/src/common/utils/db/redis.ts b/nodejs/src/common/utils/db/redis.tsindex cfe7b7ce6fd0..f6aa01137904 100644--- a/nodejs/src/common/utils/db/redis.ts+++ b/nodejs/src/common/utils/db/redis.ts@@ -9,6 +9,7 @@ import { captureException } from '../posthog'/** Number of Redis error events until the server is killed gracefully. */const REDIS_ERROR_COUNTER_LIMIT = 10+const NON_FATAL_CONNECT_TIMEOUT_MS = 10_000/*** Configuration for a Redis connection.@@ -19,6 +20,12 @@ export interface RedisConnectionConfig {url: stringoptions?: RedisOptionsname?: string+ /**+ * Stops the process after repeated connection errors. Defaults to true. Set it to false for an+ * optional store, so an outage of that store does not take down every route of the service.+ * A client that cannot connect is then closed, and its caller sees the error.+ */+ exitOnRepeatedErrors?: boolean}/**@@ -38,7 +45,7 @@ export interface RedisPoolConfig {}export async function createRedisFromConfig(config: RedisConnectionConfig): Promise<Redis.Redis> {- return createRedisClient(config.url, config.options, config.name)+ return createRedisClient(config.url, config.options, config.name, config.exitOnRepeatedErrors ?? true)}export function createRedisPoolFromConfig(config: RedisPoolConfig): RedisPool {@@ -99,7 +106,8 @@ export function getRedisHost(url: string, options?: RedisOptions): string {export async function createRedisClient(url: string,options?: RedisOptions,- connectionName?: string+ connectionName?: string,+ exitOnRepeatedErrors: boolean = true): Promise<Redis.Redis> {const redis = new Redis(url, {...options,@@ -112,8 +120,11 @@ export async function createRedisClient(redis.on('error', (error) => {errorCounter++- captureException(error)- if (errorCounter > REDIS_ERROR_COUNTER_LIMIT) {+ // A non-fatal client reconnects on every error, so only its first error is reported.+ if (exitOnRepeatedErrors || errorCounter === 1) {+ captureException(error)+ }+ if (exitOnRepeatedErrors && errorCounter > REDIS_ERROR_COUNTER_LIMIT) {logger.error('😡',`${connectionId}Redis error encountered! host: ${redisHost} Enough of this, I quit!`,@@ -133,6 +144,28 @@ export async function createRedisClient(logger.info('✅', `${connectionId}Connected to Redis!`, redisHost)}})- await redis.info()+ try {+ if (exitOnRepeatedErrors) {+ await redis.info()+ } else {+ // A command waits offline while the client reconnects, so commandTimeout never starts for it.+ let timer: NodeJS.Timeout | undefined+ await Promise.race([+ redis.info(),+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error(`${connectionId}Redis did not connect`)),+ options?.commandTimeout ?? NON_FATAL_CONNECT_TIMEOUT_MS+ )+ }),+ ]).finally(() => clearTimeout(timer))+ }+ } catch (error) {+ if (!exitOnRepeatedErrors) {+ // Nothing holds this client, so it would otherwise reconnect forever.+ redis.disconnect()+ }+ throw error+ }return redis}diff --git a/nodejs/src/server.ts b/nodejs/src/server.tsindex 3f5d995b513c..81391973e04b 100644--- a/nodejs/src/server.ts+++ b/nodejs/src/server.ts@@ -45,6 +45,7 @@ import { CyclotronV2JanitorService, CyclotronV2Manager, CyclotronV2Worker } fromimport { HogFlowScheduleService } from './cdp/services/hogflow-schedule/hogflow-schedule.service'import { HOGFLOW_BATCH_RESOLVE_QUEUE } from './cdp/services/hogflows/batch-resolver.types'import { HogFlowBatchPersonQueryService } from './cdp/services/hogflows/hogflow-batch-person-query.service'+import { createIdempotencyValkeyPool } from './cdp/services/hogflows/idempotency-keys'import { CyclotronJobQueueKafka } from './cdp/services/job-queue/job-queue-kafka'import { CyclotronJobQueuePostgresV2 } from './cdp/services/job-queue/job-queue-postgres-v2'import { CyclotronJobQueueRateLimitedPostgresV2 } from './cdp/services/job-queue/job-queue-rate-limited-postgres-v2'@@ -244,7 +245,7 @@ export class PluginServer implements NodeServer {: nullconst api = new CdpApi(this.config,- cdpDeps!,+ { ...cdpDeps!, idempotencyValkey: createIdempotencyValkeyPool(this.config) },{ hogQueue: kafkaQueue, hogflowQueue: postgresV2Queue },batchResolverProducer)