Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): keep retrying a failed idempotency completion in the background (#123481)

Harley Alexander6ebffe237973
diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts
index 0b9b90b30946..d98fbb69e639 100644
--- a/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts
+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.test.ts
@@ -1,5 +1,6 @@
import Redis from 'ioredis'
import { register } from 'prom-client'
+import { setTimeout as delay } from 'timers/promises'
import { defaultConfig } from '~/common/config/config'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
@@ -7,9 +8,12 @@ import * as redisDb from '~/common/utils/db/redis'
import { createRedisClient } from '~/common/utils/db/redis'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'
+import { waitForExpect } from '~/tests/helpers/expectations'
import { IdempotencyKeys, createIdempotencyValkeyPool } from './idempotency-keys'
+const HOUR_MS = 60 * 60 * 1000
+
describe('IdempotencyKeys', () => {
// Alerts read this counter, so an outcome that stops being counted silences them.
const operations = async (scope: string, operation: string, outcome: string): Promise<number> =>
@@ -64,6 +68,143 @@ describe('IdempotencyKeys', () => {
expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'replayed', invocationId: 'run-1' })
})
+ it('records the run once the store recovers from an outage that outlasts the inline attempts', async () => {
+ await keys.reserve(1, key, 'request-a', 'run-1')
+ const failedBefore = await operations('test', 'mark_queued', 'failed')
+ const useClient = valkey.useClient
+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))
+ try {
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+ expect(await operations('test', 'mark_queued', 'failed')).toEqual(failedBefore + 1)
+ const inlineAttempts = outage.mock.calls.length
+ await waitForExpect(() => expect(outage.mock.calls.length).toBeGreaterThan(inlineAttempts))
+ outage.mockImplementation(useClient)
+
+ await waitForExpect(async () =>
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({
+ status: 'replayed',
+ invocationId: 'run-1',
+ })
+ )
+ } finally {
+ outage.mockRestore()
+ }
+ })
+
+ it('keeps the replay window of the run when the store recovers hours later', async () => {
+ await keys.reserve(1, key, 'request-a', 'run-1')
+ const useClient = valkey.useClient
+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))
+ const realNow = Date.now
+ const clock = jest.spyOn(Date, 'now')
+ try {
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+ clock.mockImplementation(() => realNow() + 12 * HOUR_MS)
+ outage.mockImplementation(useClient)
+
+ await waitForExpect(async () =>
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({
+ status: 'replayed',
+ invocationId: 'run-1',
+ })
+ )
+ const ttlSeconds = await valkey.useClient({ name: 'test-ttl' }, (client) =>
+ client.ttl(keys['redisKey'](1, key))
+ )
+ expect(ttlSeconds).toBeGreaterThan((12 * HOUR_MS) / 1000 - 60)
+ expect(ttlSeconds).toBeLessThanOrEqual((12 * HOUR_MS) / 1000)
+ } finally {
+ outage.mockRestore()
+ clock.mockRestore()
+ }
+ })
+
+ it('records a retry whose completion fails while the earlier completion of its key is still writing', async () => {
+ await keys.reserve(1, key, 'request-a', 'run-1')
+ const useClient = valkey.useClient
+ const connectionLost = new Error('connection lost')
+ let endSlowWrite!: () => void
+ const slowWrite = new Promise<void>((resolve) => (endSlowWrite = resolve))
+ const store = jest
+ .spyOn(valkey, 'useClient')
+ .mockRejectedValueOnce(connectionLost)
+ .mockRejectedValueOnce(connectionLost)
+ .mockImplementationOnce(async (...args) => {
+ await slowWrite
+ return useClient(...args)
+ })
+ .mockImplementationOnce(useClient)
+ .mockImplementationOnce(useClient)
+ .mockRejectedValueOnce(connectionLost)
+ .mockRejectedValueOnce(connectionLost)
+ .mockImplementation(useClient)
+ try {
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+ await waitForExpect(() => expect(store.mock.calls.length).toEqual(3))
+ // The pending hold of run-1 expires, and a retry of the request reserves the key again.
+ await valkey.useClient({ name: 'test-expire' }, (client) => client.del(keys['redisKey'](1, key)))
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'reserved' })
+ await keys.markQueued(1, key, 'request-a', 'run-2')
+ endSlowWrite()
+
+ await waitForExpect(async () =>
+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({
+ status: 'replayed',
+ invocationId: 'run-2',
+ })
+ )
+ } finally {
+ store.mockRestore()
+ }
+ })
+
+ it('records the reserved run when a retry that queued without a reservation also fails to complete', async () => {
+ expect(await keys.reserve(1, key, 'request-a', 'run-1')).toEqual({ status: 'reserved' })
+ const useClient = valkey.useClient
+ const outage = jest.spyOn(valkey, 'useClient').mockRejectedValue(new Error('connection lost'))
+ try {
+ await keys.markQueued(1, key, 'request-a', 'run-1')
+ // The webhook route queues a retry without a reservation when the store is unavailable.
+ expect(await keys.reserve(1, key, 'request-a', 'run-2')).toEqual({ status: 'unavailable' })
+ await keys.markQueued(1, key, 'request-a', 'run-2')
+ // The store recovers before the pending hold of run-1 expires.
+ outage.mockImplementation(useClient)
+
+ await waitForExpect(async () =>
+ expect(await keys.reserve(1, key, 'request-a', 'run-3')).toEqual({
+ status: 'replayed',
+ invocationId: 'run-1',
+ })
+ )
+ } finally {
+ outage.mockRestore()
+ }
+ })
+
+ it('sends one store call at a time while failed completions wait for the store', async () => {
+ let inFlight = 0
+ let maxInFlight = 0
+ const outage = jest.spyOn(valkey, 'useClient').mockImplementation(async () => {
+ inFlight++
+ maxInFlight = Math.max(maxInFlight, inFlight)
+ await delay(50)
+ inFlight--
+ throw new Error('connection lost')
+ })
+ try {
+ await Promise.all(
+ Array.from({ length: 20 }, (_, i) => keys.markQueued(1, `${key}-${i}`, 'request-a', `run-${i}`))
+ )
+ maxInFlight = 0
+ const inlineAttempts = outage.mock.calls.length
+ await waitForExpect(() => expect(outage.mock.calls.length).toBeGreaterThanOrEqual(inlineAttempts + 2))
+
+ expect(maxInFlight).toEqual(1)
+ } finally {
+ outage.mockRestore()
+ }
+ })
+
it('reports the store as unavailable when it is not configured', async () => {
const unavailableBefore = await operations('unconfigured', 'reserve', 'unavailable')
diff --git a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts
index 96d81a786cc3..cd4a0e78480a 100644
--- a/nodejs/src/cdp/services/hogflows/idempotency-keys.ts
+++ b/nodejs/src/cdp/services/hogflows/idempotency-keys.ts
@@ -1,5 +1,6 @@
import { createHash } from 'crypto'
import { Counter, Histogram } from 'prom-client'
+import { setTimeout as delay } from 'timers/promises'
import type { CommonConfig } from '~/common/config'
import { RedisClient, RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
@@ -8,8 +9,8 @@ import { logger } from '~/common/utils/logger'
import type { CdpConfig } from '../../config'
-// Covers a caller's timeout and retry. After this a pending key expires, so a crash between the
-// reservation and the queue write can cost a duplicate, but never a send the caller thinks happened.
+// Covers a caller's timeout and retry. After this a pending key expires, so a crash after the queue
+// write and before the run is recorded can cost a duplicate, but never a send the caller thinks happened.
const PENDING_TTL_SECONDS = 60
const QUEUED_TTL_SECONDS = 24 * 60 * 60
// A hung Valkey must not hold the caller's request open. Each route decides what an unavailable store means.
@@ -20,7 +21,10 @@ const RELEASE_SCRIPT = `local v = redis.call('GET', KEYS[1]) if v and cjson.deco
// Records the queued run when the key is absent or still holds this request's reservation. The key is
// absent when the queue write took longer than the pending TTL, and the run is queued all the same.
const MARK_QUEUED_SCRIPT = `local v = redis.call('GET', KEYS[1]) if (not v) or cjson.decode(v).invocationId == ARGV[2] then redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[3]) return 1 end return 0`
-const MARK_QUEUED_ATTEMPTS = 2
+const MARK_QUEUED_INLINE_ATTEMPTS = 2
+const MARK_QUEUED_RETRY_FIRST_DELAY_MS = 250
+const MARK_QUEUED_RETRY_MAX_DELAY_MS = 5000
+const MAX_PENDING_COMPLETIONS = 10_000
const idempotencyOperationsTotal = new Counter({
name: 'cdp_hogflow_idempotency_operations_total',
@@ -119,12 +123,22 @@ export type IdempotencyReservation =
type StoredKey = { fingerprint: string; invocationId: string; queued: boolean }
+type PendingCompletion = { redisKey: string; value: StoredKey; expiresAt: number }
+
+// Keyed by run, not by Redis key. Several runs of one key can wait at once, for example a webhook
+// retry that queued without a reservation during the outage. Only the store knows which run owns
+// the key, so every run keeps its entry until its own write succeeds, loses ownership, or expires.
+const pendingCompletionKey = ({ redisKey, value }: PendingCompletion): string => `${redisKey}/${value.invocationId}`
+
/**
* Caller-supplied idempotency keys, scoped to a team. A key reserves a run before it is queued, and
* holds the run's id for 24 hours once it is. A queue failure releases the key, so the caller's
* retry can queue the run.
*/
export class IdempotencyKeys {
+ private pendingCompletions = new Map<string, PendingCompletion>()
+ private draining = false
+
constructor(
private valkey: RedisV2 | null,
private scope: string
@@ -190,33 +204,16 @@ export class IdempotencyKeys {
}
async markQueued(teamId: number, key: string, fingerprint: string, invocationId: string): Promise<void> {
- const value: StoredKey = { fingerprint, invocationId, queued: true }
- for (let attempt = 1; attempt <= MARK_QUEUED_ATTEMPTS; attempt++) {
+ const completion: PendingCompletion = {
+ redisKey: this.redisKey(teamId, key),
+ value: { fingerprint, invocationId, queued: true },
+ expiresAt: Date.now() + QUEUED_TTL_SECONDS * 1000,
+ }
+ for (let attempt = 1; attempt <= MARK_QUEUED_INLINE_ATTEMPTS; attempt++) {
try {
- const written = await this.timed(
- 'mark_queued',
- () =>
- this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>
- client.eval(
- MARK_QUEUED_SCRIPT,
- 1,
- this.redisKey(teamId, key),
- JSON.stringify(value),
- invocationId,
- QUEUED_TTL_SECONDS
- )
- ) ?? Promise.resolve(null)
- )
- if (written === 0) {
- logger.warn('🔑', 'idempotency key held by another request, run not recorded', {
- scope: this.scope,
- })
- }
- this.count('mark_queued', written === 0 ? 'not_owned' : 'written')
+ await this.writeQueued(completion)
return
} catch (error) {
- // The run is already queued, so the request still succeeds. Without this record a retry
- // after the pending TTL queues the run again.
logger.error('🔑', 'idempotency key completion failed', {
scope: this.scope,
attempt,
@@ -225,6 +222,67 @@ export class IdempotencyKeys {
}
}
this.count('mark_queued', 'failed')
+ this.retryInBackground(completion)
+ }
+
+ private retryInBackground(completion: PendingCompletion): void {
+ if (this.pendingCompletions.size >= MAX_PENDING_COMPLETIONS) {
+ logger.error('🔑', 'idempotency key completion dropped, too many waiting for the store', {
+ scope: this.scope,
+ })
+ this.count('mark_queued', 'dropped')
+ return
+ }
+ this.pendingCompletions.set(pendingCompletionKey(completion), completion)
+ if (!this.draining) {
+ void this.drainPendingCompletions()
+ }
+ }
+
+ // One loop per store sends one call at a time, so a long outage does not end in a burst that
+ // starves the reservations of new requests.
+ private async drainPendingCompletions(): Promise<void> {
+ this.draining = true
+ let delayMs = MARK_QUEUED_RETRY_FIRST_DELAY_MS
+ while (this.pendingCompletions.size > 0) {
+ await delay(delayMs, undefined, { ref: false })
+ const drained = await this.writePendingCompletions()
+ delayMs = drained ? MARK_QUEUED_RETRY_FIRST_DELAY_MS : Math.min(delayMs * 2, MARK_QUEUED_RETRY_MAX_DELAY_MS)
+ }
+ this.draining = false
+ }
+
+ private async writePendingCompletions(): Promise<boolean> {
+ for (const completion of this.pendingCompletions.values()) {
+ if (completion.expiresAt <= Date.now()) {
+ // A record after the replay window serves no retry.
+ logger.error('🔑', 'idempotency key completion gave up', { scope: this.scope })
+ this.count('mark_queued', 'expired')
+ } else {
+ try {
+ await this.writeQueued(completion)
+ } catch {
+ return false
+ }
+ }
+ this.pendingCompletions.delete(pendingCompletionKey(completion))
+ }
+ return true
+ }
+
+ private async writeQueued({ redisKey, value, expiresAt }: PendingCompletion): Promise<void> {
+ const ttlSeconds = Math.ceil((expiresAt - Date.now()) / 1000)
+ const written = await this.timed(
+ 'mark_queued',
+ () =>
+ this.valkey?.useClient({ name: 'idempotency-mark-queued' }, (client) =>
+ client.eval(MARK_QUEUED_SCRIPT, 1, redisKey, JSON.stringify(value), value.invocationId, ttlSeconds)
+ ) ?? Promise.resolve(null)
+ )
+ if (written === 0) {
+ logger.warn('🔑', 'idempotency key held by another request, run not recorded', { scope: this.scope })
+ }
+ this.count('mark_queued', written === 0 ? 'not_owned' : 'written')
}
async release(teamId: number, key: string, invocationId: string): Promise<void> {