Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

17.0 KB
Raw Blame
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch } from '~/tests/helpers/mocks/request.mock'

import { Server } from 'http'
import supertest from 'supertest'
import express from 'ultimate-express'

import { HOG_EXAMPLES, HOG_FILTERS_EXAMPLES, HOG_INPUTS_EXAMPLES } from '~/cdp/_tests/examples'
import { insertHogFunction as _insertHogFunction, insertBatchExport } from '~/cdp/_tests/fixtures'
import { CdpApi } from '~/cdp/cdp-api'
import { HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
import { GroupReadRepository } from '~/common/groups/repositories/group-repository.interface'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { createTestTeamFixture, updateOrganizationAvailableFeatures } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'

import { GroupTypeIndex, TeamId } from '../../types'
import { GroupsManagerService } from './managers/groups-manager.service'

describe('BatchExportHogFunctionService', () => {
    let hub: Hub
    let team: Team
    let api: CdpApi
    let app: express.Application
    let server: Server

    let batchExportId: string
    let hogFunction: HogFunctionType
    let clickhouseEvent: Record<string, any>

    const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
        const item = await _insertHogFunction(hub.postgres, team.id, hogFunction)
        api['hogFunctionManager']['onHogFunctionsReloaded'](team.id, [item.id])
        return item
    }

    const invocationUrl = () => `/api/projects/${team.id}/hog_functions/${hogFunction.id}/batch_export_invocations`

    const postInvocation = (body: any) => supertest(app).post(invocationUrl()).send(body)

    beforeAll(async () => {
        hub = await createHub({ SITE_URL: 'http://localhost:8000' })

        api = new CdpApi(hub, createCdpConsumerDeps(hub), {
            hogQueue: createMockJobQueue(),
            hogflowQueue: createMockJobQueue(),
        })
        app = setupExpressApp()
        app.use('/', api.router())
        server = app.listen(0, () => {})
    })

    beforeEach(async () => {
        team = (await createTestTeamFixture(hub.postgres)).team
        mockFetch.mockClear()

        clickhouseEvent = {
            uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
            event: '$pageview',
            team_id: team.id,
            distinct_id: '123',
            timestamp: '2021-09-28T14:00:00Z',
            created_at: '2021-09-28T14:00:00Z',
            properties: JSON.stringify({ $lib_version: '1.0.0' }),
            elements_chain: '',
        }

        batchExportId = new UUIDT().toString()
        await insertBatchExport(hub.postgres, team.id, batchExportId)

        hogFunction = await insertHogFunction({
            name: 'test batch export hog function',
            ...HOG_EXAMPLES.simple_fetch,
            ...HOG_INPUTS_EXAMPLES.simple_fetch,
            ...HOG_FILTERS_EXAMPLES.no_filters,
            batch_export_id: batchExportId,
        })
    })

    afterAll(async () => {
        await api.stop()
        server.close()
        await closeHub(hub)
    })

    describe('request body validation', () => {
        it.each([
            ['empty body', {}, 'clickhouse_event'],
            [
                'missing event uuid',
                { clickhouse_event: { event: '$pageview', team_id: 1, distinct_id: 'x', timestamp: 't' } },
                'uuid',
            ],
            [
                'missing event name',
                {
                    clickhouse_event: {
                        uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                        team_id: 1,
                        distinct_id: 'x',
                        timestamp: 't',
                    },
                },
                'event',
            ],
        ])('rejects %s', async (_label, body, expectedField) => {
            const res = await postInvocation(body)
            expect(res.status).toEqual(400)
            expect(res.body.errors[0]).toContain('Invalid request body')
            expect(res.body.errors[0]).toContain(expectedField)
        })

        it('rejects non-string invocation_id', async () => {
            const res = await postInvocation({ clickhouse_event: clickhouseEvent, invocation_id: 12345 })
            expect(res.status).toEqual(400)
            expect(res.body.errors[0]).toContain('Invalid request body')
            expect(res.body.errors[0]).toContain('invocation_id')
        })

        it('rejects invalid uuid invocation_id', async () => {
            const res = await postInvocation({ clickhouse_event: clickhouseEvent, invocation_id: 'not-a-uuid' })
            expect(res.status).toEqual(400)
            expect(res.body.errors[0]).toContain('Invalid request body')
            expect(res.body.errors[0]).toContain('invocation_id')
        })
    })

    describe('resource lookup errors', () => {
        it('returns 404 for non-existent team', async () => {
            const res = await supertest(app)
                .post(`/api/projects/99999/hog_functions/${hogFunction.id}/batch_export_invocations`)
                .send({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(404)
            expect(res.body.errors[0]).toContain('99999')
        })

        it('returns 404 for non-existent hog function', async () => {
            const fakeId = new UUIDT().toString()
            const res = await supertest(app)
                .post(`/api/projects/${team.id}/hog_functions/${fakeId}/batch_export_invocations`)
                .send({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(404)
            expect(res.body.errors[0]).toContain(fakeId)
        })

        it('returns 404 for hog function without batch_export_id', async () => {
            const nonBatchFunction = await insertHogFunction({
                name: 'non-batch hog function',
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_INPUTS_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
            })

            const res = await supertest(app)
                .post(`/api/projects/${team.id}/hog_functions/${nonBatchFunction.id}/batch_export_invocations`)
                .send({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(404)
            expect(res.body.errors[0]).toContain(nonBatchFunction.id)
        })
    })

    describe('successful invocation', () => {
        it('executes the hog function and returns success', async () => {
            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)
            expect(res.body.status).toEqual('success')
            expect(res.body.errors).toEqual([])
            expect(res.body.logs).toMatchObject([
                { level: 'info', message: expect.stringContaining('Fetch response:') },
                { level: 'debug', message: expect.stringContaining('Function completed in') },
            ])
        })

        it('uses provided invocation_id', async () => {
            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const invocationId = new UUIDT().toString()
            const res = await postInvocation({
                clickhouse_event: clickhouseEvent,
                invocation_id: invocationId,
            })

            expect(res.status).toEqual(200)
            expect(res.body.status).toEqual('success')
        })

        it('generates globals with source metadata', async () => {
            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)

            expect(mockFetch).toHaveBeenCalledTimes(1)
            const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
            expect(fetchBody).toMatchObject({
                event: expect.objectContaining({
                    uuid: 'b3a1fe86-b10c-43cc-acaf-d208977608d0',
                    event: '$pageview',
                    distinct_id: '123',
                }),
            })
        })

        it('produces monitoring metrics', async () => {
            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            await postInvocation({ clickhouse_event: clickhouseEvent })
            await api['batchExportHogFunctionService'].stop()

            const metrics = mockProducerObserver
                .getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
                .map((x) => x.value) as any[]

            expect(metrics).toEqual(
                expect.arrayContaining([
                    expect.objectContaining({
                        metric_name: 'succeeded',
                    }),
                ])
            )
        })
    })

    describe('execution errors', () => {
        it('returns a failed invocation, not a 500, when an integration was deleted', async () => {
            hogFunction = await insertHogFunction({
                ...HOG_EXAMPLES.simple_fetch,
                ...HOG_FILTERS_EXAMPLES.no_filters,
                batch_export_id: batchExportId,
                inputs_schema: [
                    ...(HOG_INPUTS_EXAMPLES.simple_fetch.inputs_schema ?? []),
                    { key: 'oauth', type: 'integration', required: true },
                ],
                inputs: { ...HOG_INPUTS_EXAMPLES.simple_fetch.inputs, oauth: { value: 99999999 } },
            })

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)
            expect(res.body.status).toEqual('error')
            expect(res.body.errors).toEqual([
                expect.stringContaining("The integration for input 'oauth' no longer exists"),
            ])
            expect(mockFetch).not.toHaveBeenCalled()
        })

        it('returns error log when fetch returns 500', async () => {
            mockFetch.mockImplementation(() =>
                Promise.resolve({
                    status: 500,
                    headers: { 'Content-Type': 'text/plain' },
                    json: () => Promise.reject(new Error('not json')),
                    text: () => Promise.resolve('Internal Server Error'),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)
            expect(res.body.logs).toEqual(
                expect.arrayContaining([
                    expect.objectContaining({
                        level: 'error',
                        message: expect.stringContaining('HTTP fetch failed'),
                    }),
                ])
            )
        })
    })

    describe('groups enrichment', () => {
        let originalGroupsManager: GroupsManagerService

        beforeEach(() => {
            originalGroupsManager = api['batchExportHogFunctionService']['groupsManager']
        })

        afterEach(() => {
            api['batchExportHogFunctionService']['groupsManager'] = originalGroupsManager
        })

        const setupGroups = async () => {
            await updateOrganizationAvailableFeatures(hub.postgres, team.organization_id, [
                { key: 'data_pipelines', name: 'Data Pipelines' },
                { key: 'group_analytics', name: 'Group Analytics' },
            ])
            hub.teamManager['lazyLoader'].clear()

            const mockGroupRepo: GroupReadRepository = {
                fetchGroupsByKeys: jest.fn().mockResolvedValue([
                    {
                        team_id: team.id as TeamId,
                        group_type_index: 0 as GroupTypeIndex,
                        group_key: 'acme-inc',
                        group_properties: { name: 'Acme Inc', industry: 'Tech' },
                    },
                ]),
                fetchGroupTypesByTeamIds: jest.fn().mockImplementation((teamIds: number[]) => {
                    const result: Record<string, { group_type: string; group_type_index: GroupTypeIndex }[]> = {}
                    for (const id of teamIds) {
                        result[id.toString()] = [{ group_type: 'company', group_type_index: 0 as GroupTypeIndex }]
                    }
                    return Promise.resolve(result)
                }),
                fetchGroupTypesByProjectIds: jest.fn().mockResolvedValue({}),
            }

            api['batchExportHogFunctionService']['groupsManager'] = new GroupsManagerService(
                hub.teamManager,
                mockGroupRepo
            )
        }

        it('enriches globals with group properties when event has $groups', async () => {
            await setupGroups()

            clickhouseEvent.properties = JSON.stringify({
                $lib_version: '1.0.0',
                $groups: { company: 'acme-inc' },
            })

            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)
            expect(res.body.status).toEqual('success')

            const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
            expect(fetchBody.groups).toMatchObject({
                company: expect.objectContaining({
                    id: 'acme-inc',
                    type: 'company',
                    properties: { name: 'Acme Inc', industry: 'Tech' },
                }),
            })
        })

        it('returns empty groups when event has no $groups property', async () => {
            await setupGroups()

            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)

            const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
            expect(fetchBody.groups).toEqual({})
        })

        it('returns empty groups when team lacks group_analytics feature', async () => {
            clickhouseEvent.properties = JSON.stringify({
                $lib_version: '1.0.0',
                $groups: { company: 'acme-inc' },
            })

            mockFetch.mockImplementationOnce(() =>
                Promise.resolve({
                    status: 200,
                    headers: { 'Content-Type': 'application/json' },
                    json: () => Promise.resolve({ ok: true }),
                    text: () => Promise.resolve(JSON.stringify({ ok: true })),
                    dump: () => Promise.resolve(),
                })
            )

            const res = await postInvocation({ clickhouse_event: clickhouseEvent })

            expect(res.status).toEqual(200)

            const fetchBody = parseJSON(mockFetch.mock.calls[0][1].body)
            expect(fetchBody.groups).toEqual({})
        })
    })
})