Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): idempotency key on the webhook trigger (#117287)

jake sciotto0899a94ab2a9
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 2dbf45ac3e9c..5868cd1dcb56 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -226,7 +226,12 @@ export class CdpApi {
})
this.hogQueue = jobQueues.hogQueue
this.hogflowQueue = jobQueues.hogflowQueue
- this.cdpSourceWebhooksConsumer = new CdpSourceWebhooksConsumer(config, deps, jobQueues)
+ this.cdpSourceWebhooksConsumer = new CdpSourceWebhooksConsumer(
+ config,
+ deps,
+ jobQueues,
+ new IdempotencyKeys(deps.idempotencyValkey ?? null, 'workflow-webhook')
+ )
this.hogFunctionMonitoringService = services.hogFunctionMonitoringService
this.emailTrackingService = new EmailTrackingService(
this.hogFunctionManager,
diff --git a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts
index 5985a30f8633..40b403243202 100644
--- a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts
@@ -14,17 +14,20 @@ import { template as pixelTemplate } from '~/cdp/templates/_sources/pixel/pixel.
import { template as incomingWebhookTemplate } from '~/cdp/templates/_sources/webhook/incoming_webhook.template'
import { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult, HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
+import { defaultConfig } from '~/common/config/config'
import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
+import { waitForExpect } from '~/tests/helpers/expectations'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
import { insertHogFlow } from '../_tests/fixtures-hogflows'
import { getCustomHttpResponse } from '../consumers/cdp-source-webhooks.consumer'
+import { createIdempotencyValkeyPool } from '../services/hogflows/idempotency-keys'
import { HogWatcherState } from '../services/monitoring/hog-watcher.service'
import { compileHog } from '../templates/compiler'
import { compileInputs } from '../templates/test/test-helpers'
@@ -89,6 +92,8 @@ describe('getCustomHttpResponse', () => {
})
describe('SourceWebhooksConsumer', () => {
+ // One pool for the file, because a pool for each test leaves open connections behind.
+ const idempotencyValkey = createIdempotencyValkeyPool(defaultConfig)
let hub: Hub
let team: Team
@@ -118,10 +123,14 @@ describe('SourceWebhooksConsumer', () => {
beforeEach(async () => {
hub.CDP_WATCHER_OBSERVE_RESULTS_BUFFER_TIME_MS = 50
- api = new CdpApi(hub, createCdpConsumerDeps(hub), {
- hogQueue: createMockJobQueue(),
- hogflowQueue: createMockJobQueue(),
- })
+ api = new CdpApi(
+ hub,
+ { ...createCdpConsumerDeps(hub), idempotencyValkey },
+ {
+ hogQueue: createMockJobQueue(),
+ hogflowQueue: createMockJobQueue(),
+ }
+ )
mockExecuteSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogExecutorAsync'], 'execute')
mockQueueInvocationsSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogQueue'], 'queueInvocations')
mockQueueHogflowInvocationsSpy = jest.spyOn(
@@ -593,6 +602,114 @@ describe('SourceWebhooksConsumer', () => {
expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)).toEqual([])
})
+ describe('idempotency key', () => {
+ const postWithKey = (webhookId: string, idempotencyKey: string) =>
+ doPostRequest({
+ webhookId,
+ headers: { 'Idempotency-Key': idempotencyKey },
+ body: { distinct_id: 'test-distinct-id' },
+ })
+
+ const insertSecondHogFlow = async (): Promise<HogFlow> => {
+ const other = new FixtureHogFlowBuilder()
+ .withTeamId(team.id)
+ .withSimpleWorkflow({ trigger: hogFlow.trigger })
+ .build()
+ await insertHogFlow(hub.postgres, other)
+ return other
+ }
+
+ it('queues one run and answers duplicate for a repeated key on the same workflow', async () => {
+ hub.HOG_INVOCATION_RESULTS_ENABLED = true
+ const first = await postWithKey(hogFlow.id, 'escalation-1')
+ const second = await postWithKey(hogFlow.id, 'escalation-1')
+
+ expect(first.status).toEqual(201)
+ expect(first.body.status).toEqual('queued')
+ expect(second.status).toEqual(200)
+ expect(second.body.status).toEqual('duplicate')
+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
+
+ await waitForBackgroundTasks()
+ expect(
+ mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)
+ ).toHaveLength(1)
+ expect(getLogs()).toContainEqual(
+ expect.stringContaining('Skipped duplicate request for Idempotency-Key "escalation-1"')
+ )
+ })
+
+ it('queues a run on each workflow that receives the same key', async () => {
+ const otherHogFlow = await insertSecondHogFlow()
+
+ const first = await postWithKey(hogFlow.id, 'escalation-2')
+ const second = await postWithKey(otherHogFlow.id, 'escalation-2')
+
+ expect(first.status).toEqual(201)
+ expect(second.status).toEqual(201)
+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
+ })
+
+ it('releases the key when queueing fails so a retry queues the run', async () => {
+ mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
+
+ const failed = await postWithKey(hogFlow.id, 'escalation-3')
+ const retried = await postWithKey(hogFlow.id, 'escalation-3')
+
+ expect(failed.status).toEqual(500)
+ expect(retried.status).toEqual(201)
+ expect(retried.body.status).toEqual('queued')
+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
+ })
+
+ it('tells a retry to wait while the first request is still queueing', async () => {
+ let finishQueueing: () => void = () => {}
+ mockQueueHogflowInvocationsSpy.mockImplementationOnce(
+ () => new Promise<void>((resolve) => (finishQueueing = resolve))
+ )
+ const first = postWithKey(hogFlow.id, 'escalation-5').then((res) => res)
+ await waitForExpect(() => expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1))
+
+ const retry = await postWithKey(hogFlow.id, 'escalation-5')
+ finishQueueing()
+
+ expect(retry.status).toEqual(409)
+ expect(retry.body.status).toEqual('in_flight')
+ expect((await first).status).toEqual(201)
+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)
+ })
+
+ it('queues the run when the idempotency store never answers, and releases it if queueing fails', async () => {
+ const valkey = api['cdpSourceWebhooksConsumer']['idempotencyKeys']['valkey']!
+ const useClient = valkey.useClient
+ let reservations = 0
+ jest.spyOn(valkey, 'useClient').mockImplementation((options, callback) => {
+ if (options.name === 'idempotency-reserve' && reservations++ === 0) {
+ void useClient(options, callback)
+ return new Promise(() => {})
+ }
+ return useClient(options, callback)
+ })
+ mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))
+
+ const failed = await postWithKey(hogFlow.id, 'escalation-4')
+ const retried = await postWithKey(hogFlow.id, 'escalation-4')
+
+ expect(failed.status).toEqual(500)
+ expect(retried.status).toEqual(201)
+ expect(retried.body.status).toEqual('queued')
+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)
+ })
+
+ it('rejects a key longer than 255 characters without queueing', async () => {
+ const res = await postWithKey(hogFlow.id, 'a'.repeat(256))
+
+ expect(res.status).toEqual(400)
+ expect(res.body.error).toEqual('Idempotency-Key must be 255 characters or fewer')
+ expect(mockQueueHogflowInvocationsSpy).not.toHaveBeenCalled()
+ })
+ })
+
it('does not report a workflow trigger as CDP usage', async () => {
const reportBillableInvocation = jest.spyOn(
api['cdpSourceWebhooksConsumer']['cdpUsageReporter'],
diff --git a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts
index f8bbc4746a4e..715dde0abf62 100644
--- a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts
@@ -10,6 +10,7 @@ import { UUID, UUIDT } from '~/common/utils/utils'
import { HealthCheckResult, HealthCheckResultOk, PluginsServerConfig } from '../../types'
import { createHogFlowInvocation } from '../services/hogflows/hogflow-executor.service'
import { actionIdForLogging } from '../services/hogflows/hogflow-utils'
+import { IdempotencyKeys } from '../services/hogflows/idempotency-keys'
import { JobQueue } from '../services/job-queue/job-queue.interface'
import { HogWatcherFunctionState, HogWatcherState, sameWatcherState } from '../services/monitoring/hog-watcher.service'
import {
@@ -40,6 +41,8 @@ const DISALLOWED_HEADERS = [
'true-client-ip',
]
+const IDEMPOTENCY_KEY_MAX_LENGTH = 255
+
const getFirstHeaderValue = (value: string | string[] | undefined): string | undefined => {
return Array.isArray(value) ? value[0] : value
}
@@ -110,7 +113,8 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf
constructor(
config: PluginsServerConfig,
deps: CdpConsumerBaseDeps,
- jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue }
+ jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue },
+ private idempotencyKeys: IdempotencyKeys = new IdempotencyKeys(null, 'workflow-webhook')
) {
super(config, deps)
this.promiseScheduler = new PromiseScheduler()
@@ -211,11 +215,32 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf
}
}
+ private getIdempotencyKey(req: ModifiedRequest, hogFlow: HogFlow): string | null {
+ if (hogFlow.trigger?.type !== 'webhook') {
+ return null
+ }
+ const idempotencyKey = getFirstHeaderValue(req.headers['idempotency-key'])
+ if (!idempotencyKey) {
+ return null
+ }
+ if (idempotencyKey.length > IDEMPOTENCY_KEY_MAX_LENGTH) {
+ throw new SourceWebhookError(
+ 400,
+ `Idempotency-Key must be ${IDEMPOTENCY_KEY_MAX_LENGTH} characters or fewer`
+ )
+ }
+ return idempotencyKey
+ }
+
private async executeHogFlow(
req: ModifiedRequest,
hogFlow: HogFlow,
hogFunction: HogFunctionType
): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {
+ const idempotencyKey = this.getIdempotencyKey(req, hogFlow)
+ // A key belongs to one workflow, so the same key on two workflows starts a run on each.
+ const scopedIdempotencyKey = idempotencyKey ? `${hogFlow.id}/${idempotencyKey}` : null
+
logger.info('Executing hog flow trigger', {
id: hogFlow.id,
template_id: hogFunction.template_id,
@@ -312,6 +337,35 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf
hogFlowInvocation.id = invocationId // Keep the IDs consistent
+ if (scopedIdempotencyKey) {
+ // Every repeat of a key on the workflow is the same request, so the fingerprint is the workflow.
+ const reservation = await this.idempotencyKeys.reserve(
+ hogFlow.team_id,
+ scopedIdempotencyKey,
+ hogFlow.id,
+ invocationId
+ )
+ if (reservation.status === 'replayed') {
+ addLog('info', `Skipped duplicate request for Idempotency-Key "${idempotencyKey}"`)
+ functionResult.finished = false
+ functionResult.execResult = { httpResponse: { status: 200, body: { status: 'duplicate' } } }
+ return functionResult
+ }
+ if (reservation.status === 'in_flight') {
+ // The first request can still fail to queue, so this one must not report a duplicate.
+ functionResult.finished = false
+ functionResult.execResult = { httpResponse: { status: 409, body: { status: 'in_flight' } } }
+ return functionResult
+ }
+ if (reservation.status === 'unavailable') {
+ // A webhook run matters more than its deduplication, so the run queues without a claim.
+ logger.warn('Idempotency store unavailable, queueing the hog flow run anyway', {
+ team_id: hogFlow.team_id,
+ hog_flow_id: hogFlow.id,
+ })
+ }
+ }
+
addMetric({
metric_kind: 'other',
metric_name: 'triggered',
@@ -326,7 +380,22 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf
'running'
)
- await this.hogflowQueue.queueInvocations([hogFlowInvocation])
+ try {
+ await this.hogflowQueue.queueInvocations([hogFlowInvocation])
+ } catch (error) {
+ if (scopedIdempotencyKey) {
+ await this.idempotencyKeys.release(hogFlow.team_id, scopedIdempotencyKey, invocationId)
+ }
+ throw error
+ }
+ if (scopedIdempotencyKey) {
+ await this.idempotencyKeys.markQueued(
+ hogFlow.team_id,
+ scopedIdempotencyKey,
+ hogFlow.id,
+ invocationId
+ )
+ }
} else {
addMetric({
metric_kind: 'failure',