Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

40.8 KB
Raw Blame
import { MockKafkaProducerWrapper } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'

import { KafkaProducerObserver } from '~/tests/helpers/mocks/producer.spy'

import { createHmac } from 'node:crypto'

import { KAFKA_APP_METRICS_2, KAFKA_LOG_ENTRIES } from '~/common/config/kafka-topics'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
import { TEST_KAFKA_TOPICS, ensureKafkaTopics } from '~/tests/helpers/kafka'
import { forSnapshot } from '~/tests/helpers/snapshots'
import { getFirstTeam, resetTestDatabase } from '~/tests/helpers/sql'

import { CdpCyclotronWorker } from '../../src/cdp/consumers/cdp-cyclotron-worker.consumer'
import { CdpDatawarehouseEventsConsumer } from '../../src/cdp/consumers/cdp-data-warehouse-events.consumer'
import { HogFunctionInvocationGlobals, HogFunctionType } from '../../src/cdp/types'
import { Hub, Team } from '../../src/types'
import { HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from './_tests/examples'
import {
    insertHogFunction as _insertHogFunction,
    createHogExecutionGlobals,
    insertIntegration,
} from './_tests/fixtures'
import { CdpEventsConsumer } from './consumers/cdp-events.consumer'
import { CyclotronJobQueueKafka } from './services/job-queue/job-queue-kafka'
import { CyclotronJobQueuePostgresV2 } from './services/job-queue/job-queue-postgres-v2'
import { template as webhookTemplate } from './templates/_destinations/webhook/webhook.template'
import { compileHog } from './templates/compiler'

const ActualKafkaProducerWrapper = jest.requireActual('~/common/kafka/producer').KafkaProducerWrapper

describe('CDP Consumer loop', () => {
    jest.setTimeout(20000)

    describe('e2e fetch call', () => {
        let eventsConsumer: CdpEventsConsumer
        let cyclotronWorker: CdpCyclotronWorker

        let hub: Hub
        let kafkaProducer: KafkaProducerWrapper = undefined as unknown as KafkaProducerWrapper
        let team: Team
        let fnFetchNoFilters: HogFunctionType
        let globals: HogFunctionInvocationGlobals
        let mockProducerObserver: KafkaProducerObserver

        const insertHogFunction = async (hogFunction: Partial<HogFunctionType>): Promise<HogFunctionType> => {
            const item = await _insertHogFunction(hub.postgres, team.id, hogFunction)
            return item
        }

        beforeEach(async () => {
            // Each `KafkaProducerWrapper.create()` call (e.g. inside each cyclotron job
            // queue) gets a fresh real producer it owns and disconnects on stop. The
            // CDP outputs registry uses the dedicated `kafkaProducer` below — a separate
            // real producer the test owns, observes, and disconnects in `afterEach`.
            MockKafkaProducerWrapper.create = jest.fn((...args) => {
                return ActualKafkaProducerWrapper.create(...args)
            })

            await ensureKafkaTopics(TEST_KAFKA_TOPICS)

            await resetTestDatabase()
            hub = await createHub()

            kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
            mockProducerObserver = new KafkaProducerObserver(kafkaProducer)

            team = await getFirstTeam(hub.postgres)
            mockProducerObserver.resetKafkaProducer()

            hub.CDP_FETCH_RETRIES = 2
            hub.CDP_FETCH_BACKOFF_BASE_MS = 100 // fast backoff
            hub.CDP_CYCLOTRON_COMPRESS_KAFKA_DATA = true

            // Include integration parsing as part of the e2e check
            await insertIntegration(hub.postgres, team.id, {
                id: 1,
                kind: 'slack',
                config: {},
                sensitive_config: {
                    access_token: hub.encryptedFields.encrypt('super-secret-token'),
                },
            })

            const hog = `
            let res := fetch(inputs.url, {
                'headers': {
                  'Authorization': f'Bearer {inputs.oauth.access_token}',
                },
                'body': inputs.body,
                'method': inputs.method
            });

            print('Fetch response:', res);
            `

            fnFetchNoFilters = await insertHogFunction({
                type: 'destination',
                hog: hog,
                bytecode: await compileHog(hog),
                inputs_schema: [
                    ...(HOG_INPUTS_EXAMPLES.simple_fetch.inputs_schema ?? []),
                    { key: 'oauth', type: 'integration', label: 'Slack', secret: false, required: true },
                ],
                inputs: {
                    ...HOG_INPUTS_EXAMPLES.simple_fetch.inputs,
                    oauth: {
                        value: 1,
                    },
                },
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
            const postgresV2Queue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)

            eventsConsumer = new CdpEventsConsumer(hub, createCdpConsumerDeps(hub, kafkaProducer), {
                hogQueue: kafkaQueue,
                hogflowQueue: postgresV2Queue,
            })
            await eventsConsumer.start()

            cyclotronWorker = new CdpCyclotronWorker(hub, createCdpConsumerDeps(hub, kafkaProducer), kafkaQueue)
            await cyclotronWorker.start()

            globals = createHogExecutionGlobals({
                project: {
                    id: team.id,
                } as any,
                event: {
                    uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                    event: '$pageview',
                    properties: {
                        $current_url: 'https://posthog.com',
                        $lib_version: '1.0.0',
                    },
                    timestamp: '2024-09-03T09:00:00Z',
                } as any,
            })

            mockFetch.mockResolvedValue({
                status: 200,
                json: () => Promise.resolve({ success: true }),
                text: () => Promise.resolve(JSON.stringify({ success: true })),
                headers: { 'Content-Type': 'application/json' },
                dump: () => Promise.resolve(),
            })

            expect(mockProducerObserver.getProducedKafkaMessages()).toHaveLength(0)
        })

        afterEach(async () => {
            await Promise.all([
                eventsConsumer?.stop().then(() => console.log('Stopped eventsConsumer')),
                cyclotronWorker?.stop().then(() => console.log('Stopped cyclotronWorker')),
            ])
            await kafkaProducer.disconnect()
            await closeHub(hub)
            mockProducerObserver.resetKafkaProducer()
        })

        afterAll(() => {
            jest.useRealTimers()
        })

        /**
         * Tests here are somewhat expensive so should mostly simulate happy paths and the more e2e scenarios
         */

        it('should invoke a function in the worker loop until completed', async () => {
            const { invocations } = await eventsConsumer.processBatch([globals])
            expect(invocations).toHaveLength(1)

            try {
                await waitForExpect(() => {
                    expect(mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')).toHaveLength(2)
                }, 5000)
            } catch (e) {
                logger.warn('[TESTS] Failed to wait for log messages', {
                    messages: mockProducerObserver.getProducedKafkaMessages(),
                })
                throw e
            }

            expect(mockFetch).toHaveBeenCalledTimes(1)

            expect(mockFetch.mock.calls[0]).toMatchInlineSnapshot(`
                [
                  "https://example.com/posthog-webhook",
                  {
                    "body": "{"event":{"uuid":"b3a1fe86-b10c-43cc-acaf-d208977608d0","event":"$pageview","elements_chain":"","distinct_id":"distinct_id","url":"http://localhost:8000/events/1","properties":{"$current_url":"https://posthog.com","$lib_version":"1.0.0"},"timestamp":"2024-09-03T09:00:00Z"},"groups":{},"nested":{"foo":"http://localhost:8000/events/1"},"person":{"id":"uuid","name":"test","url":"http://localhost:8000/persons/1","properties":{"email":"[email protected]","first_name":"Pumpkin"}},"event_url":"http://localhost:8000/events/1-test"}",
                    "headers": {
                      "Authorization": "Bearer super-secret-token",
                    },
                    "maxResponseBytes": 5242880,
                    "method": "POST",
                  },
                ]
            `)

            const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
            const metricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)

            expect(metricsMessages).toMatchObject([
                {
                    topic: 'clickhouse_app_metrics2_test',
                    value: {
                        app_source: 'hog_function',
                        app_source_id: fnFetchNoFilters.id.toString(),
                        count: 1,
                        metric_kind: 'other',
                        metric_name: 'triggered',
                        team_id: 2,
                    },
                },
                {
                    topic: 'clickhouse_app_metrics2_test',
                    value: {
                        app_source: 'hog_function',
                        app_source_id: '_event_trigger',
                        count: 1,
                        metric_kind: 'billing',
                        metric_name: 'billable_invocation',
                        team_id: 2,
                    },
                },
                {
                    topic: 'clickhouse_app_metrics2_test',
                    value: {
                        app_source: 'hog_function',
                        app_source_id: fnFetchNoFilters.id.toString(),
                        count: 1,
                        metric_kind: 'other',
                        metric_name: 'fetch',
                        team_id: 2,
                    },
                },
                {
                    topic: 'clickhouse_app_metrics2_test',
                    value: {
                        app_source: 'hog_function',
                        app_source_id: fnFetchNoFilters.id.toString(),
                        count: 1,
                        metric_kind: 'success',
                        metric_name: 'succeeded',
                        team_id: 2,
                    },
                },
            ])

            expect(logMessages).toMatchObject([
                {
                    topic: 'log_entries_test',
                    value: {
                        level: 'info',
                        log_source: 'hog_function',
                        log_source_id: fnFetchNoFilters.id.toString(),
                        message: `Fetch response:, {"status":200,"body":{"success":true}}`,
                        team_id: 2,
                    },
                },
                {
                    topic: 'log_entries_test',
                    value: {
                        level: 'debug',
                        log_source: 'hog_function',
                        log_source_id: fnFetchNoFilters.id.toString(),
                        message: expect.stringContaining('Function completed in'),
                        team_id: 2,
                    },
                },
            ])
        })

        // Regression coverage for the production wire shape of an outgoing Kinesis
        // PutRecord. The pre-refactor Python template test
        // (products/cdp/backend/templates/aws_kinesis/test_template_aws_kinesis.py) asserted a
        // similar shape but ran against the Python Hog VM, whose `jsonStringify`
        // formats objects WITH spaces (`{"k": "v"}`). The Node Hog VM — what CDP
        // actually runs in production — formats WITHOUT spaces (`{"k":"v"}`), so the
        // Python test was locking in a body the Node runtime never produced. This
        // e2e test uses the Node values (body + corresponding signature) so a
        // regression in Hog body construction, executor wiring, credential lookup,
        // or signer canonical-request handling will all flip the signature hex and
        // fail this test.
        it('matches the production outgoing request shape for the kinesis template fixture', async () => {
            const fixedTime = new Date('2024-04-16T12:34:51Z').getTime()
            const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(fixedTime)

            try {
                mockFetch.mockResolvedValueOnce({
                    status: 200,
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve('{"ok":true}'),
                    headers: { 'Content-Type': 'application/json' },
                    dump: () => Promise.resolve(),
                })

                // Same Hog body the production Kinesis template produces, parameterised
                // off `inputs` so the test exercises the input lookup path end-to-end.
                const hog = `
                let payload := jsonStringify({
                  'StreamName': inputs.aws_kinesis_stream_name,
                  'PartitionKey': inputs.aws_kinesis_partition_key ?? generateUUIDv4(),
                  'Data': base64Encode(jsonStringify(inputs.payload)),
                })

                let res := fetch(f'https://kinesis.{inputs.aws_region}.amazonaws.com', {
                  'method': 'POST',
                  'headers': {
                    'Content-Type': 'application/x-amz-json-1.1',
                    'X-Amz-Target': 'Kinesis_20131202.PutRecord',
                  },
                  'body': payload,
                  'aws_sigv4': {
                    'service': 'kinesis',
                    'region': inputs.aws_region,
                    'access_key_id_input': 'aws_access_key_id',
                    'secret_access_key_input': 'aws_secret_access_key',
                  },
                });

                print('Fetch response:', res);
                `

                await insertHogFunction({
                    type: 'destination',
                    hog,
                    bytecode: await compileHog(hog),
                    inputs_schema: [
                        { key: 'aws_access_key_id', type: 'string', label: 'AKID', secret: true, required: true },
                        { key: 'aws_secret_access_key', type: 'string', label: 'SK', secret: true, required: true },
                        { key: 'aws_region', type: 'string', label: 'region', secret: false, required: true },
                        {
                            key: 'aws_kinesis_stream_name',
                            type: 'string',
                            label: 'stream',
                            secret: false,
                            required: true,
                        },
                        {
                            key: 'aws_kinesis_partition_key',
                            type: 'string',
                            label: 'partition',
                            secret: false,
                            required: false,
                        },
                        { key: 'payload', type: 'json', label: 'payload', secret: false, required: true },
                    ],
                    inputs: {
                        aws_region: { value: 'aws_region' },
                        aws_kinesis_stream_name: { value: 'aws_kinesis_stream_arn' },
                        aws_kinesis_partition_key: { value: '1' },
                        payload: { value: { hello: 'world' } },
                    },
                    encrypted_inputs: hub.encryptedFields.encrypt(
                        JSON.stringify({
                            aws_access_key_id: { value: 'aws_access_key_id' },
                            aws_secret_access_key: { value: 'aws_secret_access_key' },
                        })
                    ),
                    ...HOG_FILTERS_EXAMPLES.no_filters,
                } as any)

                await eventsConsumer.processBatch([globals])

                await waitForExpect(() => {
                    const kinesisCalls = mockFetch.mock.calls.filter((c: any[]) => String(c[0]).includes('kinesis.'))
                    expect(kinesisCalls.length).toBe(1)
                }, 5000)

                const kinesisCall = mockFetch.mock.calls.find((c: any[]) => String(c[0]).includes('kinesis.'))
                expect(kinesisCall).toBeDefined()
                const [url, opts] = kinesisCall as [string, any]

                // Body bytes the Node Hog VM emits for this input (no spaces between
                // JSON tokens), and the SigV4 signature derived from THOSE bytes plus
                // the frozen `20240416T123451Z` timestamp. The byte-compatibility of
                // the signer itself (matching the canonical AWS docs algorithm) is
                // separately locked in by the `aws-sigv4.test.ts` unit fixture.
                expect(url).toBe('https://kinesis.aws_region.amazonaws.com')
                expect(opts).toMatchObject({
                    method: 'POST',
                    body: '{"StreamName":"aws_kinesis_stream_arn","PartitionKey":"1","Data":"eyJoZWxsbyI6IndvcmxkIn0="}',
                    headers: {
                        'Content-Type': 'application/x-amz-json-1.1',
                        'X-Amz-Target': 'Kinesis_20131202.PutRecord',
                        'X-Amz-Date': '20240416T123451Z',
                        Host: 'kinesis.aws_region.amazonaws.com',
                        Authorization:
                            'AWS4-HMAC-SHA256 Credential=aws_access_key_id/20240416/aws_region/kinesis/aws4_request, ' +
                            'SignedHeaders=content-type;host;x-amz-date;x-amz-target, ' +
                            'Signature=19caaa3f67e2214daf7c0318dd0fa92ccf94d94208aabf526b82f8990ed2607b',
                    },
                })
            } finally {
                dateSpy.mockRestore()
            }
        })

        // E2E coverage for the AWS SigV4 fetch-time signing path. The regression this
        // guards against: secrets land in `encrypted_inputs` after `move_secret_inputs`
        // runs on save, so the executor has to look them up there — not in `inputs`,
        // which is empty for `secret: true` keys. If we ever break that lookup the
        // upstream gets an unsigned request and AWS 401s. Also covers retry — the
        // second attempt must carry a freshly-signed Authorization header, not a
        // reused stale one from the first attempt.
        it('should sign aws sigv4 fetches from encrypted_inputs on both initial attempt and retry', async () => {
            const ACCESS_KEY = 'AKIDEXAMPLE'
            const SECRET_KEY = 'wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY'

            const sigv4FetchCalls: Array<{ url: string; opts: any }> = []
            let kinesisCallCount = 0
            mockFetch.mockImplementation((url: string, opts: any) => {
                if (url.includes('kinesis.')) {
                    sigv4FetchCalls.push({ url, opts })
                    kinesisCallCount++
                    if (kinesisCallCount === 1) {
                        // Delay the first 500 by long enough that attempt-2 signs in a
                        // different wall-clock second. X-Amz-Date is second-resolution,
                        // so without this delay both attempts would land in the same
                        // second and produce identical signatures — masking whether the
                        // retry actually re-signed or just reused the original payload.
                        return new Promise((resolve) =>
                            setTimeout(
                                () =>
                                    resolve({
                                        status: 500,
                                        json: () => Promise.resolve({}),
                                        text: () => Promise.resolve(''),
                                        headers: {},
                                        dump: () => Promise.resolve(),
                                    }),
                                1500
                            )
                        )
                    }
                }
                return Promise.resolve({
                    status: 200,
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve('{"ok":true}'),
                    headers: { 'Content-Type': 'application/json' },
                    dump: () => Promise.resolve(),
                })
            })

            const hog = `
            let res := fetch('https://kinesis.us-east-1.amazonaws.com', {
                'method': 'POST',
                'headers': {
                    'Content-Type': 'application/x-amz-json-1.1',
                    'X-Amz-Target': 'Kinesis_20131202.PutRecord',
                },
                'body': '{"StreamName":"s","PartitionKey":"p","Data":"ZA=="}',
                'aws_sigv4': {
                    'service': 'kinesis',
                    'region': 'us-east-1',
                    'access_key_id_input': 'aws_access_key_id',
                    'secret_access_key_input': 'aws_secret_access_key',
                },
            });
            print('Fetch response:', res);
            `

            await insertHogFunction({
                type: 'destination',
                hog,
                bytecode: await compileHog(hog),
                inputs_schema: [
                    { key: 'aws_access_key_id', type: 'string', label: 'AKID', secret: true, required: true },
                    { key: 'aws_secret_access_key', type: 'string', label: 'SK', secret: true, required: true },
                ],
                inputs: {},
                // Mirror what Django's `move_secret_inputs` produces in prod: secret
                // inputs are Fernet-encrypted on `encrypted_inputs`. The Node manager
                // detects the string form and decrypts back to the plaintext map shape
                // before the executor sees the function.
                encrypted_inputs: hub.encryptedFields.encrypt(
                    JSON.stringify({
                        aws_access_key_id: { value: ACCESS_KEY },
                        aws_secret_access_key: { value: SECRET_KEY },
                    })
                ),
                ...HOG_FILTERS_EXAMPLES.no_filters,
            } as any)

            // The default `fnFetchNoFilters` from beforeEach also matches this event;
            // we expect both to fire, but we only inspect the Kinesis ones.
            await eventsConsumer.processBatch([globals])

            await waitForExpect(() => {
                expect(sigv4FetchCalls.length).toBeGreaterThanOrEqual(2)
            }, 5000).catch((e) => {
                logger.warn('[TESTS] Expected two Kinesis fetch attempts (initial + retry)', {
                    sigv4FetchCount: sigv4FetchCalls.length,
                    allFetchCalls: mockFetch.mock.calls.length,
                })
                throw e
            })

            // Every attempt (including the retry) must carry a fresh SigV4 Authorization
            // header derived from the decrypted `encrypted_inputs` map. A regression in
            // the lookup path would either crash with the "input not found" error or
            // ship an unsigned request — both visible here.
            const sigv4Authorizations = sigv4FetchCalls.slice(0, 2).map(({ opts }) => {
                const headers = (opts?.headers ?? {}) as Record<string, string>
                return headers['Authorization'] ?? headers.authorization
            })
            const sigv4AmzDates = sigv4FetchCalls.slice(0, 2).map(({ opts }) => {
                const headers = (opts?.headers ?? {}) as Record<string, string>
                return headers['X-Amz-Date'] ?? headers['x-amz-date']
            })

            for (const auth of sigv4Authorizations) {
                expect(auth).toMatch(
                    /^AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE\/\d{8}\/us-east-1\/kinesis\/aws4_request, SignedHeaders=[a-z0-9;-]+, Signature=[a-f0-9]{64}$/
                )
            }

            // A retry must NOT reuse the signature from the first attempt. The 1.5s
            // delay on the 500 response forces the two attempts to sign in different
            // wall-clock seconds, so a correct implementation produces different
            // X-Amz-Date values and therefore different signatures. A regression to
            // "sign once, reuse on retry" makes these match and AWS returns
            // InvalidSignatureException whenever the retry crosses the 5-minute
            // signature window.
            expect(sigv4AmzDates[0]).not.toEqual(sigv4AmzDates[1])
            expect(sigv4Authorizations[0]).not.toEqual(sigv4Authorizations[1])
        })

        // E2E coverage for Standard Webhooks signing, driven by the shipped webhook
        // template rather than hand-written Hog. Three things only reachable here:
        // `signing_secret` is a secret input, so in production it exists solely in
        // Fernet-encrypted `encrypted_inputs`; the signing payload now carries
        // `webhook_id`, so it has to survive the cyclotron serialization round-trip;
        // and the signature has to cover the exact body bytes the Node Hog VM emits.
        // The unit tests build `queueParameters` in memory and never see any of it.
        it('should sign webhook template fetches so a receiver can verify them', async () => {
            // Secret from the Standard Webhooks spec's reference example.
            const SIGNING_SECRET = 'whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw'
            const SIGNED_URL = 'https://signed.example.com/hooks'

            const fixedTime = new Date('2024-04-16T12:34:51Z').getTime()
            const dateSpy = jest.spyOn(Date, 'now').mockReturnValue(fixedTime)

            try {
                await insertHogFunction({
                    type: 'destination',
                    hog: webhookTemplate.code,
                    bytecode: await compileHog(webhookTemplate.code),
                    inputs_schema: webhookTemplate.inputs_schema,
                    inputs: {
                        url: { value: SIGNED_URL },
                        method: { value: 'POST' },
                        body: { value: { event: '{event.event}' } },
                        headers: { value: { 'Content-Type': 'application/json' } },
                    },
                    // Mirror what Django's `move_secret_inputs` produces on save: a
                    // `secret: true` input is absent from `inputs` entirely, so the
                    // executor can only find it by decrypting `encrypted_inputs`.
                    encrypted_inputs: hub.encryptedFields.encrypt(
                        JSON.stringify({ signing_secret: { value: SIGNING_SECRET } })
                    ),
                    ...HOG_FILTERS_EXAMPLES.no_filters,
                } as any)

                // The default `fnFetchNoFilters` from beforeEach also matches this
                // event, so select the signed call by URL.
                const signedCalls = (): any[][] => mockFetch.mock.calls.filter((c: any[]) => c[0] === SIGNED_URL)

                await eventsConsumer.processBatch([globals])

                await waitForExpect(() => {
                    expect(signedCalls()).toHaveLength(1)
                }, 5000)

                const [, opts] = signedCalls()[0] as [string, any]
                const headers = opts.headers as Record<string, string>

                expect(headers['webhook-id']).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)
                expect(headers['webhook-timestamp']).toBe('1713270891')
                expect(headers['Content-Type']).toBe('application/json')

                // Verify exactly the way a receiver's library does: recompute the HMAC
                // over the body bytes that arrived, keyed with the decoded secret. A
                // signature over anything but the sent bytes fails here.
                const key = Buffer.from(SIGNING_SECRET.replace('whsec_', ''), 'base64')
                const digest = createHmac('sha256', key)
                    .update(`${headers['webhook-id']}.${headers['webhook-timestamp']}.${opts.body as string}`)
                    .digest('base64')
                expect(headers['webhook-signature']).toBe(`v1,${digest}`)
            } finally {
                dateSpy.mockRestore()
            }
        })

        // E2E coverage for the webhook template's `secret_headers` input. In production the dictionary
        // exists only in Fernet-encrypted `encrypted_inputs`, the fetch queue carries an input-key
        // reference, and the executor resolves it again on every attempt. The unit tests seed
        // `encrypted_inputs` in memory and never cross the queue, so a reference dropped during
        // serialization, or a merge moved back into the Hog code, is only visible here.
        it('should send webhook template secret headers from encrypted_inputs on the first attempt and the retry', async () => {
            const TOKEN = 'sk_e2e_secret_headers_token_0000'
            const API_KEY = 'e2e-secret-headers-api-key-0000'
            const SECRET_HEADERS_URL = 'https://secret-headers.example.com/hooks'

            let secretHeaderAttempts = 0
            mockFetch.mockImplementation((url: string) => {
                const status = url === SECRET_HEADERS_URL && ++secretHeaderAttempts === 1 ? 500 : 200
                return Promise.resolve({
                    status,
                    json: () => Promise.resolve({}),
                    text: () => Promise.resolve('{}'),
                    headers: { 'Content-Type': 'application/json' },
                    dump: () => Promise.resolve(),
                })
            })

            const fn = await insertHogFunction({
                type: 'destination',
                hog: webhookTemplate.code,
                bytecode: await compileHog(webhookTemplate.code),
                inputs_schema: webhookTemplate.inputs_schema,
                inputs: {
                    url: { value: SECRET_HEADERS_URL },
                    method: { value: 'POST' },
                    body: { value: { event: '{event.event}' } },
                    // Same name as a secret header in another case, so the merge has to replace it
                    // instead of sending both.
                    headers: {
                        value: { 'Content-Type': 'application/json', authorization: 'plaintext-authorization' },
                    },
                    debug: { value: true },
                },
                encrypted_inputs: hub.encryptedFields.encrypt(
                    JSON.stringify({
                        secret_headers: { value: { Authorization: `Bearer ${TOKEN}`, 'X-Api-Key': API_KEY } },
                    })
                ),
                ...HOG_FILTERS_EXAMPLES.no_filters,
            } as any)

            // The default `fnFetchNoFilters` from beforeEach also matches this event, so select by URL.
            const secretHeaderCalls = (): any[][] =>
                mockFetch.mock.calls.filter((c: any[]) => c[0] === SECRET_HEADERS_URL)

            await eventsConsumer.processBatch([globals])

            await waitForExpect(() => {
                expect(secretHeaderCalls()).toHaveLength(2)
            }, 5000)

            for (const [, opts] of secretHeaderCalls() as [string, any][]) {
                const headers = opts.headers as Record<string, string>
                expect(headers).toMatchObject({
                    Authorization: `Bearer ${TOKEN}`,
                    'X-Api-Key': API_KEY,
                    'Content-Type': 'application/json',
                })
                expect(Object.keys(headers).filter((name) => name.toLowerCase() === 'authorization')).toEqual([
                    'Authorization',
                ])
            }

            await waitForExpect(() => {
                const messages = mockProducerObserver
                    .getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                    .filter((m: any) => m.value.log_source_id === fn.id)
                    .map((m: any) => m.value.message as string)
                expect(messages).toContainEqual(
                    expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.')
                )
                expect(messages).toContainEqual(expect.stringContaining('Response, 200'))
                const requestLogs = messages.filter((message) => message.startsWith('Request, '))
                expect(requestLogs).toHaveLength(1)
                expect(requestLogs[0]).toContain('"secret_headers_input":"secret_headers"')
                // Log redaction masks values but not header names. The name is what appears if the
                // template merges the secret headers into `payload.headers` instead of passing the reference.
                expect(requestLogs[0]).not.toContain('X-Api-Key')
                expect(JSON.stringify(messages)).not.toContain(TOKEN)
            }, 5000)
        })

        it('should handle fetch failures with retries', async () => {
            mockFetch.mockImplementation(() => {
                return Promise.resolve({
                    status: 500,
                    headers: {},
                    json: () => Promise.resolve({ error: 'Server error' }),
                    text: () => Promise.resolve(JSON.stringify({ error: 'Server error' })),
                    dump: () => Promise.resolve(),
                })
            })

            const { invocations } = await eventsConsumer.processBatch([globals])

            expect(invocations).toHaveLength(1)

            await waitForExpect(() => {
                expect(mockProducerObserver.getProducedKafkaMessages().length).toBeGreaterThan(3)
            }, 5000).catch((e) => {
                logger.warn('[TESTS] Failed to wait for log messages', {
                    messages: mockProducerObserver.getProducedKafkaMessages(),
                })
                throw e
            })

            const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)

            // Ignore the last message as it is non-deterministic
            expect(
                forSnapshot(
                    logMessages
                        .map((m) => m.value.message)
                        // Sorted compare as the messages can get logged in different orders
                        .sort()
                )
            ).toEqual([
                'Fetch response:, {"status":500,"body":{"error":"Server error"}}',
                expect.stringContaining('Function completed in '),
                expect.stringContaining('HTTP fetch failed on attempt 1 with status code 500. Retrying.'),
                // CDP_FETCH_RETRIES is 2 here, so attempt 2 is the last one and no retry follows it.
                // Matched exactly: a substring match would also accept the ' Retrying.' this guards against.
                'HTTP fetch failed on attempt 2 with status code 500.',
            ])
        })
    })

    describe('e2e data warehouse destination', () => {
        let dwhConsumer: CdpDatawarehouseEventsConsumer
        let cyclotronWorker: CdpCyclotronWorker

        let hub: Hub
        let kafkaProducer: KafkaProducerWrapper = undefined as unknown as KafkaProducerWrapper
        let team: Team
        let fnDataWarehouseDestination: HogFunctionType
        let mockProducerObserver: KafkaProducerObserver

        const TABLE_NAME = 'postgres.orders'

        const insertHogFunction = async (hogFunction: Partial<HogFunctionType>): Promise<HogFunctionType> => {
            return await _insertHogFunction(hub.postgres, team.id, hogFunction)
        }

        /** Row-scoped globals the DWH consumer produces for a synced warehouse row. */
        const createDwhGlobals = (rowProperties: Record<string, any> = {}): HogFunctionInvocationGlobals =>
            createHogExecutionGlobals({
                project: { id: team.id } as any,
                event: {
                    uuid: new UUIDT().toString(),
                    event: '$warehouse_source_row',
                    distinct_id: '',
                    properties: { ...rowProperties, $source_table: TABLE_NAME },
                    timestamp: '2024-09-03T09:00:00Z',
                } as any,
            })

        beforeEach(async () => {
            MockKafkaProducerWrapper.create = jest.fn((...args) => {
                return ActualKafkaProducerWrapper.create(...args)
            })

            await ensureKafkaTopics(TEST_KAFKA_TOPICS)
            await resetTestDatabase()
            hub = await createHub()

            kafkaProducer = await ActualKafkaProducerWrapper.create(hub.KAFKA_CLIENT_RACK)
            mockProducerObserver = new KafkaProducerObserver(kafkaProducer)

            team = await getFirstTeam(hub.postgres)
            mockProducerObserver.resetKafkaProducer()

            hub.CDP_FETCH_RETRIES = 2
            hub.CDP_FETCH_BACKOFF_BASE_MS = 100
            hub.CDP_CYCLOTRON_COMPRESS_KAFKA_DATA = true

            const hog = `
            let res := fetch(inputs.url, { 'body': inputs.body, 'method': inputs.method });
            print('Fetch response:', res);
            `

            // Destination subscribed to the data-warehouse-table source — only fires on synced rows.
            fnDataWarehouseDestination = await insertHogFunction({
                type: 'destination',
                hog,
                bytecode: await compileHog(hog),
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters_data_warehouse_table,
            })

            const kafkaQueue = new CyclotronJobQueueKafka(hub.KAFKA_CLIENT_RACK, hub, hub.CONSUMER_BATCH_SIZE)
            const postgresV2Queue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)

            dwhConsumer = new CdpDatawarehouseEventsConsumer(hub, createCdpConsumerDeps(hub, kafkaProducer), {
                hogQueue: kafkaQueue,
                hogflowQueue: postgresV2Queue,
            })
            await dwhConsumer.start()

            cyclotronWorker = new CdpCyclotronWorker(hub, createCdpConsumerDeps(hub, kafkaProducer), kafkaQueue)
            await cyclotronWorker.start()

            mockFetch.mockResolvedValue({
                status: 200,
                json: () => Promise.resolve({ success: true }),
                text: () => Promise.resolve(JSON.stringify({ success: true })),
                headers: { 'Content-Type': 'application/json' },
                dump: () => Promise.resolve(),
            })

            expect(mockProducerObserver.getProducedKafkaMessages()).toHaveLength(0)
        })

        afterEach(async () => {
            await Promise.all([dwhConsumer?.stop(), cyclotronWorker?.stop()])
            await kafkaProducer.disconnect()
            await closeHub(hub)
            mockProducerObserver.resetKafkaProducer()
        })

        afterAll(() => {
            jest.useRealTimers()
        })

        it('invokes a data-warehouse-sourced destination for a synced row', async () => {
            const { invocations } = await dwhConsumer.processBatch([createDwhGlobals({ order_id: 42 })])
            expect(invocations).toHaveLength(1)

            try {
                await waitForExpect(() => {
                    expect(mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')).toHaveLength(2)
                }, 5000)
            } catch (e) {
                logger.warn('[TESTS] Failed to wait for log messages', {
                    messages: mockProducerObserver.getProducedKafkaMessages(),
                })
                throw e
            }

            expect(mockFetch).toHaveBeenCalledTimes(1)
            expect(mockFetch).toHaveBeenCalledWith(
                'https://example.com/posthog-webhook',
                expect.objectContaining({ method: 'POST' })
            )

            const metricsMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
            expect(metricsMessages).toEqual(
                expect.arrayContaining([
                    expect.objectContaining({
                        topic: 'clickhouse_app_metrics2_test',
                        value: expect.objectContaining({
                            app_source: 'hog_function',
                            app_source_id: fnDataWarehouseDestination.id.toString(),
                            metric_kind: 'success',
                            metric_name: 'succeeded',
                            team_id: team.id,
                        }),
                    }),
                ])
            )
        })
    })
})