:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): idempotency key on the webhook trigger (#117287)
jake sciotto0899a94ab2a9
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 2dbf45ac3e9c..5868cd1dcb56 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -226,7 +226,12 @@ export class CdpApi {})this.hogQueue = jobQueues.hogQueuethis.hogflowQueue = jobQueues.hogflowQueue- this.cdpSourceWebhooksConsumer = new CdpSourceWebhooksConsumer(config, deps, jobQueues)+ this.cdpSourceWebhooksConsumer = new CdpSourceWebhooksConsumer(+ config,+ deps,+ jobQueues,+ new IdempotencyKeys(deps.idempotencyValkey ?? null, 'workflow-webhook')+ )this.hogFunctionMonitoringService = services.hogFunctionMonitoringServicethis.emailTrackingService = new EmailTrackingService(this.hogFunctionManager,diff --git a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.tsindex 5985a30f8633..40b403243202 100644--- a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.test.ts@@ -14,17 +14,20 @@ import { template as pixelTemplate } from '~/cdp/templates/_sources/pixel/pixel.import { template as incomingWebhookTemplate } from '~/cdp/templates/_sources/webhook/incoming_webhook.template'import { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult, HogFunctionType } from '~/cdp/types'import { setupExpressApp } from '~/common/api/router'+import { defaultConfig } from '~/common/config/config'import { KAFKA_HOG_INVOCATION_RESULTS } from '~/common/config/kafka-topics'import { closeHub, createHub } from '~/common/utils/db/hub'import { PostgresUse } from '~/common/utils/db/postgres'import { parseJSON } from '~/common/utils/json-parse'import { createCdpConsumerDeps } from '~/tests/helpers/cdp'+import { waitForExpect } from '~/tests/helpers/expectations'import { createTestTeamFixture } from '~/tests/helpers/sql'import { Hub, Team } from '~/types'import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'import { insertHogFlow } from '../_tests/fixtures-hogflows'import { getCustomHttpResponse } from '../consumers/cdp-source-webhooks.consumer'+import { createIdempotencyValkeyPool } from '../services/hogflows/idempotency-keys'import { HogWatcherState } from '../services/monitoring/hog-watcher.service'import { compileHog } from '../templates/compiler'import { compileInputs } from '../templates/test/test-helpers'@@ -89,6 +92,8 @@ describe('getCustomHttpResponse', () => {})describe('SourceWebhooksConsumer', () => {+ // One pool for the file, because a pool for each test leaves open connections behind.+ const idempotencyValkey = createIdempotencyValkeyPool(defaultConfig)let hub: Hublet team: Team@@ -118,10 +123,14 @@ describe('SourceWebhooksConsumer', () => {beforeEach(async () => {hub.CDP_WATCHER_OBSERVE_RESULTS_BUFFER_TIME_MS = 50- api = new CdpApi(hub, createCdpConsumerDeps(hub), {- hogQueue: createMockJobQueue(),- hogflowQueue: createMockJobQueue(),- })+ api = new CdpApi(+ hub,+ { ...createCdpConsumerDeps(hub), idempotencyValkey },+ {+ hogQueue: createMockJobQueue(),+ hogflowQueue: createMockJobQueue(),+ }+ )mockExecuteSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogExecutorAsync'], 'execute')mockQueueInvocationsSpy = jest.spyOn(api['cdpSourceWebhooksConsumer']['hogQueue'], 'queueInvocations')mockQueueHogflowInvocationsSpy = jest.spyOn(@@ -593,6 +602,114 @@ describe('SourceWebhooksConsumer', () => {expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)).toEqual([])})+ describe('idempotency key', () => {+ const postWithKey = (webhookId: string, idempotencyKey: string) =>+ doPostRequest({+ webhookId,+ headers: { 'Idempotency-Key': idempotencyKey },+ body: { distinct_id: 'test-distinct-id' },+ })++ const insertSecondHogFlow = async (): Promise<HogFlow> => {+ const other = new FixtureHogFlowBuilder()+ .withTeamId(team.id)+ .withSimpleWorkflow({ trigger: hogFlow.trigger })+ .build()+ await insertHogFlow(hub.postgres, other)+ return other+ }++ it('queues one run and answers duplicate for a repeated key on the same workflow', async () => {+ hub.HOG_INVOCATION_RESULTS_ENABLED = true+ const first = await postWithKey(hogFlow.id, 'escalation-1')+ const second = await postWithKey(hogFlow.id, 'escalation-1')++ expect(first.status).toEqual(201)+ expect(first.body.status).toEqual('queued')+ expect(second.status).toEqual(200)+ expect(second.body.status).toEqual('duplicate')+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)++ await waitForBackgroundTasks()+ expect(+ mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_HOG_INVOCATION_RESULTS)+ ).toHaveLength(1)+ expect(getLogs()).toContainEqual(+ expect.stringContaining('Skipped duplicate request for Idempotency-Key "escalation-1"')+ )+ })++ it('queues a run on each workflow that receives the same key', async () => {+ const otherHogFlow = await insertSecondHogFlow()++ const first = await postWithKey(hogFlow.id, 'escalation-2')+ const second = await postWithKey(otherHogFlow.id, 'escalation-2')++ expect(first.status).toEqual(201)+ expect(second.status).toEqual(201)+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)+ })++ it('releases the key when queueing fails so a retry queues the run', async () => {+ mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))++ const failed = await postWithKey(hogFlow.id, 'escalation-3')+ const retried = await postWithKey(hogFlow.id, 'escalation-3')++ expect(failed.status).toEqual(500)+ expect(retried.status).toEqual(201)+ expect(retried.body.status).toEqual('queued')+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)+ })++ it('tells a retry to wait while the first request is still queueing', async () => {+ let finishQueueing: () => void = () => {}+ mockQueueHogflowInvocationsSpy.mockImplementationOnce(+ () => new Promise<void>((resolve) => (finishQueueing = resolve))+ )+ const first = postWithKey(hogFlow.id, 'escalation-5').then((res) => res)+ await waitForExpect(() => expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1))++ const retry = await postWithKey(hogFlow.id, 'escalation-5')+ finishQueueing()++ expect(retry.status).toEqual(409)+ expect(retry.body.status).toEqual('in_flight')+ expect((await first).status).toEqual(201)+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(1)+ })++ it('queues the run when the idempotency store never answers, and releases it if queueing fails', async () => {+ const valkey = api['cdpSourceWebhooksConsumer']['idempotencyKeys']['valkey']!+ const useClient = valkey.useClient+ let reservations = 0+ jest.spyOn(valkey, 'useClient').mockImplementation((options, callback) => {+ if (options.name === 'idempotency-reserve' && reservations++ === 0) {+ void useClient(options, callback)+ return new Promise(() => {})+ }+ return useClient(options, callback)+ })+ mockQueueHogflowInvocationsSpy.mockRejectedValueOnce(new Error('queue unavailable'))++ const failed = await postWithKey(hogFlow.id, 'escalation-4')+ const retried = await postWithKey(hogFlow.id, 'escalation-4')++ expect(failed.status).toEqual(500)+ expect(retried.status).toEqual(201)+ expect(retried.body.status).toEqual('queued')+ expect(mockQueueHogflowInvocationsSpy).toHaveBeenCalledTimes(2)+ })++ it('rejects a key longer than 255 characters without queueing', async () => {+ const res = await postWithKey(hogFlow.id, 'a'.repeat(256))++ expect(res.status).toEqual(400)+ expect(res.body.error).toEqual('Idempotency-Key must be 255 characters or fewer')+ expect(mockQueueHogflowInvocationsSpy).not.toHaveBeenCalled()+ })+ })+it('does not report a workflow trigger as CDP usage', async () => {const reportBillableInvocation = jest.spyOn(api['cdpSourceWebhooksConsumer']['cdpUsageReporter'],diff --git a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.tsindex f8bbc4746a4e..715dde0abf62 100644--- a/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-source-webhooks.consumer.ts@@ -10,6 +10,7 @@ import { UUID, UUIDT } from '~/common/utils/utils'import { HealthCheckResult, HealthCheckResultOk, PluginsServerConfig } from '../../types'import { createHogFlowInvocation } from '../services/hogflows/hogflow-executor.service'import { actionIdForLogging } from '../services/hogflows/hogflow-utils'+import { IdempotencyKeys } from '../services/hogflows/idempotency-keys'import { JobQueue } from '../services/job-queue/job-queue.interface'import { HogWatcherFunctionState, HogWatcherState, sameWatcherState } from '../services/monitoring/hog-watcher.service'import {@@ -40,6 +41,8 @@ const DISALLOWED_HEADERS = ['true-client-ip',]+const IDEMPOTENCY_KEY_MAX_LENGTH = 255+const getFirstHeaderValue = (value: string | string[] | undefined): string | undefined => {return Array.isArray(value) ? value[0] : value}@@ -110,7 +113,8 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConfconstructor(config: PluginsServerConfig,deps: CdpConsumerBaseDeps,- jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue }+ jobQueues: { hogQueue: JobQueue; hogflowQueue: JobQueue },+ private idempotencyKeys: IdempotencyKeys = new IdempotencyKeys(null, 'workflow-webhook')) {super(config, deps)this.promiseScheduler = new PromiseScheduler()@@ -211,11 +215,32 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf}}+ private getIdempotencyKey(req: ModifiedRequest, hogFlow: HogFlow): string | null {+ if (hogFlow.trigger?.type !== 'webhook') {+ return null+ }+ const idempotencyKey = getFirstHeaderValue(req.headers['idempotency-key'])+ if (!idempotencyKey) {+ return null+ }+ if (idempotencyKey.length > IDEMPOTENCY_KEY_MAX_LENGTH) {+ throw new SourceWebhookError(+ 400,+ `Idempotency-Key must be ${IDEMPOTENCY_KEY_MAX_LENGTH} characters or fewer`+ )+ }+ return idempotencyKey+ }+private async executeHogFlow(req: ModifiedRequest,hogFlow: HogFlow,hogFunction: HogFunctionType): Promise<CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>> {+ const idempotencyKey = this.getIdempotencyKey(req, hogFlow)+ // A key belongs to one workflow, so the same key on two workflows starts a run on each.+ const scopedIdempotencyKey = idempotencyKey ? `${hogFlow.id}/${idempotencyKey}` : null+logger.info('Executing hog flow trigger', {id: hogFlow.id,template_id: hogFunction.template_id,@@ -312,6 +337,35 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConfhogFlowInvocation.id = invocationId // Keep the IDs consistent+ if (scopedIdempotencyKey) {+ // Every repeat of a key on the workflow is the same request, so the fingerprint is the workflow.+ const reservation = await this.idempotencyKeys.reserve(+ hogFlow.team_id,+ scopedIdempotencyKey,+ hogFlow.id,+ invocationId+ )+ if (reservation.status === 'replayed') {+ addLog('info', `Skipped duplicate request for Idempotency-Key "${idempotencyKey}"`)+ functionResult.finished = false+ functionResult.execResult = { httpResponse: { status: 200, body: { status: 'duplicate' } } }+ return functionResult+ }+ if (reservation.status === 'in_flight') {+ // The first request can still fail to queue, so this one must not report a duplicate.+ functionResult.finished = false+ functionResult.execResult = { httpResponse: { status: 409, body: { status: 'in_flight' } } }+ return functionResult+ }+ if (reservation.status === 'unavailable') {+ // A webhook run matters more than its deduplication, so the run queues without a claim.+ logger.warn('Idempotency store unavailable, queueing the hog flow run anyway', {+ team_id: hogFlow.team_id,+ hog_flow_id: hogFlow.id,+ })+ }+ }+addMetric({metric_kind: 'other',metric_name: 'triggered',@@ -326,7 +380,22 @@ export class CdpSourceWebhooksConsumer extends CdpConsumerBase<PluginsServerConf'running')- await this.hogflowQueue.queueInvocations([hogFlowInvocation])+ try {+ await this.hogflowQueue.queueInvocations([hogFlowInvocation])+ } catch (error) {+ if (scopedIdempotencyKey) {+ await this.idempotencyKeys.release(hogFlow.team_id, scopedIdempotencyKey, invocationId)+ }+ throw error+ }+ if (scopedIdempotencyKey) {+ await this.idempotencyKeys.markQueued(+ hogFlow.team_id,+ scopedIdempotencyKey,+ hogFlow.id,+ invocationId+ )+ }} else {addMetric({metric_kind: 'failure',