:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): keep retrying a failed idempotency completion in the background (#123481)
Harley Alexander6ebffe237973
diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.tsindex 0b9b90b30946..d98fbb69e639 100644--- a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts@@ -1,5 +1,6 @@import Redis from 'ioredis'import { register } from 'prom-client'+import { setTimeout as delay } from 'timers/promises'import { defaultConfig } from '~/common/config/config'import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'@@ -7,9 +8,12 @@ import * as redisDb from '~/common/utils/db/redis'import { createRedisClient } from '~/common/utils/db/redis'import { logger } from '~/common/utils/logger'import { UUIDT } from '~/common/utils/utils'+import { waitForExpect } from '~/tests/helpers/expectations'import { IdempotencyKeys, createIdempotencyValkeyPool } from './idempotency-keys'+const HOUR_MS = 60 * 60 * 1000+describe('IdempotencyKeys', () => {// Alerts read this counter, so an outcome that stops being counted silences them.const operations = async (scope: string, operation: string, outcome: string): Promise<number> =>@@ -64,6 +68,143 @@ describe('IdempotencyKeys', () => {expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'replayed', invocationId: 'run-1' })})+ it('records the run once the store recovers from an outage that outlasts the inline attempts', async () => {+ await keys.reserve(1, key, 'request-a', 'run-1')+ const failedBefore = await operations('test', 'mark_queued', 'failed')+ const useClient = valkey.useClient+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))+ try {+ await keys.markQueued(1, key, 'request-a', 'run-1')+ expect(await operations('test', 'mark_queued', 'failed')).toEqual(failedBefore + 1)+ const inlineAttempts = outage.mock.calls.length+ await waitForExpect(() => expect(outage.mock.calls.length).toBeGreaterThan(inlineAttempts))+ outage.mockImplementation(useClient)++ await waitForExpect(async () =>+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({+ status: 'replayed',+ invocationId: 'run-1',+ })+ )+ } finally {+ outage.mockRestore()+ }+ })++ it('keeps the replay window of the run when the store recovers hours later', async () => {+ await keys.reserve(1, key, 'request-a', 'run-1')+ const useClient = valkey.useClient+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))+ const realNow = Date.now+ const clock = jest.spyOn(Date, 'now')+ try {+ await keys.markQueued(1, key, 'request-a', 'run-1')+ clock.mockImplementation(() => realNow() + 12 * HOUR_MS)+ outage.mockImplementation(useClient)++ await waitForExpect(async () =>+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({+ status: 'replayed',+ invocationId: 'run-1',+ })+ )+ const ttlSeconds = await valkey.useClient({ name: 'test-ttl' }, (client) =>+ client.ttl(keys['redisKey'](1, key))+ )+ expect(ttlSeconds).toBeGreaterThan((12 * HOUR_MS) / 1000 - 60)+ expect(ttlSeconds).toBeLessThanOrEqual((12 * HOUR_MS) / 1000)+ } finally {+ outage.mockRestore()+ clock.mockRestore()+ }+ })++ it('records a retry whose completion fails while the earlier completion of its key is still writing', async () => {+ await keys.reserve(1, key, 'request-a', 'run-1')+ const useClient = valkey.useClient+ const connectionLost = new Error('connection lost')+ let endSlowWrite!: () => void+ const slowWrite = new Promise<void>((resolve) => (endSlowWrite = resolve))+ const store = jest+ .spyOn(valkey, 'useClient')+ .mockRejectedValueOnce(connectionLost)+ .mockRejectedValueOnce(connectionLost)+ .mockImplementationOnce(async (...args) => {+ await slowWrite+ return useClient(...args)+ })+ .mockImplementationOnce(useClient)+ .mockImplementationOnce(useClient)+ .mockRejectedValueOnce(connectionLost)+ .mockRejectedValueOnce(connectionLost)+ .mockImplementation(useClient)+ try {+ await keys.markQueued(1, key, 'request-a', 'run-1')+ await waitForExpect(() => expect(store.mock.calls.length).toEqual(3))+ // The pending hold of run-1 expires, and a retry of the request reserves the key again.+ await valkey.useClient({ name: 'test-expire' }, (client) => client.del(keys['redisKey'](1, key)))+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'reserved' })+ await keys.markQueued(1, key, 'request-a', 'run-2')+ endSlowWrite()++ await waitForExpect(async () =>+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({+ status: 'replayed',+ invocationId: 'run-2',+ })+ )+ } finally {+ store.mockRestore()+ }+ })++ it('records the reserved run when a retry that queued without a reservation also fails to complete', async () => {+ expect(await keys.reserve(1, key, 'request-a', 'run-1')).toEqual({ status: 'reserved' })+ const useClient = valkey.useClient+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))+ try {+ await keys.markQueued(1, key, 'request-a', 'run-1')+ // The webhook route queues a retry without a reservation when the store is unavailable.+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'unavailable' })+ await keys.markQueued(1, key, 'request-a', 'run-2')+ // The store recovers before the pending hold of run-1 expires.+ outage.mockImplementation(useClient)++ await waitForExpect(async () =>+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({+ status: 'replayed',+ invocationId: 'run-1',+ })+ )+ } finally {+ outage.mockRestore()+ }+ })++ it('sends one store call at a time while failed completions wait for the store', async () => {+ let inFlight = 0+ let maxInFlight = 0+ const outage = jest.spyOn(valkey, 'useClient').mockImplementation(async () => {+ inFlight+++ maxInFlight = Math.max(maxInFlight, inFlight)+ await delay(50)+ inFlight--+ throw new Error('connection lost')+ })+ try {+ await Promise.all(+ Array.from({ length: 20 }, (_, i) => keys.markQueued(1, `${key}-${i}`, 'request-a', `run-${i}`))+ )+ maxInFlight = 0+ const inlineAttempts = outage.mock.calls.length+ await waitForExpect(() => expect(outage.mock.calls.length).toBeGreaterThanOrEqual(inlineAttempts + 2))++ expect(maxInFlight).toEqual(1)+ } finally {+ outage.mockRestore()+ }+ })+it('reports the store as unavailable when it is not configured', async () => {const unavailableBefore = await operations('unconfigured', 'reserve', 'unavailable')diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.tsindex 96d81a786cc3..cd4a0e78480a 100644--- a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts@@ -1,5 +1,6 @@import { createHash } from 'crypto'import { Counter, Histogram } from 'prom-client'+import { setTimeout as delay } from 'timers/promises'import type { CommonConfig } from '~/common/config'import { RedisClient, RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'@@ -8,8 +9,8 @@ import { logger } from '~/common/utils/logger'import type { CdpConfig } from '../../config'-// Covers a caller's timeout and retry. After this a pending key expires, so a crash between the-// reservation and the queue write can cost a duplicate, but never a send the caller thinks happened.+// Covers a caller's timeout and retry. After this a pending key expires, so a crash after the queue+// write and before the run is recorded can cost a duplicate, but never a send the caller thinks happened.const PENDING_TTL_SECONDS = 60const QUEUED_TTL_SECONDS = 24 * 60 * 60// A hung Valkey must not hold the caller's request open. Each route decides what an unavailable store means.@@ -20,7 +21,10 @@ const RELEASE_SCRIPT = `local v = redis.call('GET', KEYS[1]) if v and cjson.deco// Records the queued run when the key is absent or still holds this request's reservation. The key is// absent when the queue write took longer than the pending TTL, and the run is queued all the same.const MARK_QUEUED_SCRIPT = `local v = redis.call('GET', KEYS[1]) if (not v) or cjson.decode(v).invocationId == ARGV[2] then redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[3]) return 1 end return 0`-const MARK_QUEUED_ATTEMPTS = 2+const MARK_QUEUED_INLINE_ATTEMPTS = 2+const MARK_QUEUED_RETRY_FIRST_DELAY_MS = 250+const MARK_QUEUED_RETRY_MAX_DELAY_MS = 5000+const MAX_PENDING_COMPLETIONS = 10_000const idempotencyOperationsTotal = new Counter({name: 'cdp_hogflow_idempotency_operations_total',@@ -119,12 +123,22 @@ export type IdempotencyReservation =type StoredKey = { fingerprint: string; invocationId: string; queued: boolean }+type PendingCompletion = { redisKey: string; value: StoredKey; expiresAt: number }++// Keyed by run, not by Redis key. Several runs of one key can wait at once, for example a webhook+// retry that queued without a reservation during the outage. Only the store knows which run owns+// the key, so every run keeps its entry until its own write succeeds, loses ownership, or expires.+const pendingCompletionKey = ({ redisKey, value }: PendingCompletion): string => `${redisKey}/${value.invocationId}`+/*** Caller-supplied idempotency keys, scoped to a team. A key reserves a run before it is queued, and* holds the run's id for 24 hours once it is. A queue failure releases the key, so the caller's* retry can queue the run.*/export class IdempotencyKeys {+ private pendingCompletions = new Map<string, PendingCompletion>()+ private draining = false+constructor(private valkey: RedisV2 | null,private scope: string@@ -190,33 +204,16 @@ export class IdempotencyKeys {}async markQueued(teamId: number, key: string, fingerprint: string, invocationId: string): Promise<void> {- const value: StoredKey = { fingerprint, invocationId, queued: true }- for (let attempt = 1; attempt <= MARK_QUEUED_ATTEMPTS; attempt++) {+ const completion: PendingCompletion = {+ redisKey: this.redisKey(teamId, key),+ value: { fingerprint, invocationId, queued: true },+ expiresAt: Date.now() + QUEUED_TTL_SECONDS * 1000,+ }+ for (let attempt = 1; attempt <= MARK_QUEUED_INLINE_ATTEMPTS; attempt++) {try {- const written = await this.timed(- 'mark_queued',- () =>- this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>- client.eval(- MARK_QUEUED_SCRIPT,- 1,- this.redisKey(teamId, key),- JSON.stringify(value),- invocationId,- QUEUED_TTL_SECONDS- )- ) ?? Promise.resolve(null)- )- if (written === 0) {- logger.warn('🔑', 'idempotency key held by another request, run not recorded', {- scope: this.scope,- })- }- this.count('mark_queued', written === 0 ? 'not_owned' : 'written')+ await this.writeQueued(completion)return} catch (error) {- // The run is already queued, so the request still succeeds. Without this record a retry- // after the pending TTL queues the run again.logger.error('🔑', 'idempotency key completion failed', {scope: this.scope,attempt,@@ -225,6 +222,67 @@ export class IdempotencyKeys {}}this.count('mark_queued', 'failed')+ this.retryInBackground(completion)+ }++ private retryInBackground(completion: PendingCompletion): void {+ if (this.pendingCompletions.size >= MAX_PENDING_COMPLETIONS) {+ logger.error('🔑', 'idempotency key completion dropped, too many waiting for the store', {+ scope: this.scope,+ })+ this.count('mark_queued', 'dropped')+ return+ }+ this.pendingCompletions.set(pendingCompletionKey(completion), completion)+ if (!this.draining) {+ void this.drainPendingCompletions()+ }+ }++ // One loop per store sends one call at a time, so a long outage does not end in a burst that+ // starves the reservations of new requests.+ private async drainPendingCompletions(): Promise<void> {+ this.draining = true+ let delayMs = MARK_QUEUED_RETRY_FIRST_DELAY_MS+ while (this.pendingCompletions.size > 0) {+ await delay(delayMs, undefined, { ref: false })+ const drained = await this.writePendingCompletions()+ delayMs = drained ? MARK_QUEUED_RETRY_FIRST_DELAY_MS : Math.min(delayMs * 2, MARK_QUEUED_RETRY_MAX_DELAY_MS)+ }+ this.draining = false+ }++ private async writePendingCompletions(): Promise<boolean> {+ for (const completion of this.pendingCompletions.values()) {+ if (completion.expiresAt <= Date.now()) {+ // A record after the replay window serves no retry.+ logger.error('🔑', 'idempotency key completion gave up', { scope: this.scope })+ this.count('mark_queued', 'expired')+ } else {+ try {+ await this.writeQueued(completion)+ } catch {+ return false+ }+ }+ this.pendingCompletions.delete(pendingCompletionKey(completion))+ }+ return true+ }++ private async writeQueued({ redisKey, value, expiresAt }: PendingCompletion): Promise<void> {+ const ttlSeconds = Math.ceil((expiresAt - Date.now()) / 1000)+ const written = await this.timed(+ 'mark_queued',+ () =>+ this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>+ client.eval(MARK_QUEUED_SCRIPT, 1, redisKey, JSON.stringify(value), value.invocationId, ttlSeconds)+ ) ?? Promise.resolve(null)+ )+ if (written === 0) {+ logger.warn('🔑', 'idempotency key held by another request, run not recorded', { scope: this.scope })+ }+ this.count('mark_queued', written === 0 ? 'not_owned' : 'written')}async release(teamId: number, key: string, invocationId: string): Promise<void> {