Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

65.6 KB
Raw Blame
// Serial: resets shared Postgres and relies on the reset-created default team.
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'

import { DecodedKafkaMessage } from '~/tests/helpers/mocks/producer.spy'

import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'

import { insertHogFunction as _insertHogFunction } from '~/cdp/_tests/fixtures'
import { createHogTransformerService } from '~/cdp/hog-transformations/hog-transformer.service'
import { template as geoipTemplate } from '~/cdp/templates/_transformations/geoip/geoip.template'
import { compileHog } from '~/cdp/templates/compiler'
import { HogFunctionType } from '~/cdp/types'
import { ClickhouseGroupRepository } from '~/common/groups/repositories/clickhouse-group-repository'
import { COOKIELESS_SENTINEL_VALUE } from '~/common/persons/person-utils'
import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'
import { COOKIELESS_MODE_FLAG_PROPERTY } from '~/ingestion/common/cookieless/cookieless-manager'
import { BatchWritingPersonsStore } from '~/ingestion/common/persons/batch-writing-person-store'
import { createPrepareEventStep } from '~/ingestion/common/steps/event-processing/prepare-event-step'
import { createProcessGroupsStep } from '~/ingestion/common/steps/event-processing/process-groups-step'
import { IngestionTestInfra, createIngestionTestInfra } from '~/tests/helpers/ingestion-e2e'
import { createTestIngestionOutputs, createTestMonitoringOutputs } from '~/tests/helpers/ingestion-outputs'
import { forSnapshot } from '~/tests/helpers/snapshots'
import { createTeam, fetchPostgresPersons, getFirstTeam, getTeam, resetTestDatabase } from '~/tests/helpers/sql'
import { CookielessServerHashMode, PipelineEvent, Team } from '~/types'

import { IngestionConsumer, latestOffsetTimestampGauge } from './ingestion-consumer'

const DEFAULT_TEST_TIMEOUT = 5000
jest.setTimeout(DEFAULT_TEST_TIMEOUT)

jest.mock('~/common/utils/posthog', () => {
    const original = jest.requireActual('~/common/utils/posthog')
    return {
        ...original,
        captureException: jest.fn(),
    }
})

// Mock the prepare event step for error testing (a step without a retry option)
jest.mock('~/ingestion/common/steps/event-processing/prepare-event-step', () => ({
    createPrepareEventStep: jest.fn(),
}))

// Mock the process groups step for error testing (a step with a retry option)
jest.mock('~/ingestion/common/steps/event-processing/process-groups-step', () => ({
    createProcessGroupsStep: jest.fn(),
}))

// Mock the IngestionWarningLimiter to always allow warnings (prevents rate limiting between tests)
jest.mock('~/common/utils/token-bucket', () => {
    const mockConsume = jest.fn().mockReturnValue(true)
    return {
        ...jest.requireActual('~/common/utils/token-bucket'),
        IngestionWarningLimiter: {
            consume: mockConsume,
        },
    }
})

let offsetIncrementer = 0

const createKafkaMessage = (event: PipelineEvent, token: string): Message => {
    // TRICKY: This is the slightly different format that capture sends
    const captureEvent = {
        uuid: event.uuid,
        distinct_id: event.distinct_id,
        ip: event.ip,
        now: event.now,
        token,
        data: JSON.stringify(event),
    }
    return {
        key: `${token}:${event.distinct_id}`,
        value: Buffer.from(JSON.stringify(captureEvent)),
        size: 1,
        topic: 'test',
        offset: offsetIncrementer++,
        timestamp: DateTime.now().toMillis(),
        partition: 1,
        headers: [
            {
                distinct_id: Buffer.from(event.distinct_id || ''),
            },
            {
                token: Buffer.from(token),
            },
            {
                event: Buffer.from(event.event || ''),
            },
            {
                uuid: Buffer.from(event.uuid || ''),
            },
            {
                now: Buffer.from(event.now || ''),
            },
        ],
    }
}

describe('IngestionConsumer', () => {
    let ingester: IngestionConsumer
    let infra: IngestionTestInfra
    let team: Team
    let team2: Team
    let fixedTime: DateTime

    const createIngestionConsumer = async (
        infra: IngestionTestInfra,
        overrides?: ConstructorParameters<typeof IngestionConsumer>[2]
    ) => {
        const outputs = createTestIngestionOutputs(mockProducer)
        const ingester = new IngestionConsumer(
            infra.config,
            {
                ...infra,
                outputs,
                clickhouseGroupRepository: new ClickhouseGroupRepository(outputs),
                hogTransformer: createHogTransformerService(infra.config, {
                    ...infra,
                    monitoringOutputs: createTestMonitoringOutputs(mockProducer),
                }),
            },
            overrides
        )
        // NOTE: We don't actually use kafka so we skip instantiation for faster tests
        ingester['kafkaConsumer'] = {
            connect: jest.fn(),
            disconnect: jest.fn(),
            isHealthy: jest.fn(),
        } as any
        await ingester.start()
        return ingester
    }

    const createEvent = (event?: Partial<PipelineEvent>): PipelineEvent => ({
        distinct_id: 'user-1',
        uuid: new UUIDT().toString(),
        ip: '127.0.0.1',
        site_url: 'us.posthog.com',
        now: fixedTime.toISO()!,
        event: '$pageview',
        ...event,
        properties: {
            $current_url: 'http://localhost:8000',
            ...(event?.properties || {}),
        },
    })

    const createCookielessEvent = (event?: Partial<PipelineEvent>): PipelineEvent => ({
        distinct_id: COOKIELESS_SENTINEL_VALUE,
        uuid: new UUIDT().toString(),
        ip: '127.0.0.1',
        site_url: 'us.posthog.com',
        now: fixedTime.toISO()!,
        event: '$pageview',
        ...event,
        properties: {
            $current_url: 'http://localhost:8000',
            $host: 'localhost:8000',
            $raw_user_agent: 'Chrome',
            $session_id: null,
            $device_id: null,
            [COOKIELESS_MODE_FLAG_PROPERTY]: true,
            ...(event?.properties || {}),
        },
    })

    const createKafkaMessages = (events: PipelineEvent[], token?: string): Message[] => {
        return events.map((event) => createKafkaMessage(event, token ?? team.api_token))
    }

    beforeEach(async () => {
        fixedTime = DateTime.fromObject({ year: 2025, month: 1, day: 1 }, { zone: 'UTC' })
        jest.spyOn(Date, 'now').mockReturnValue(fixedTime.toMillis())
        jest.spyOn(Date.prototype, 'toISOString').mockReturnValue(fixedTime.toISO()!)

        offsetIncrementer = 0
        await resetTestDatabase()
        infra = await createIngestionTestInfra()

        // mockProducer = mockProducer
        team = await getFirstTeam(infra.postgres)
        const team2Id = await createTeam(infra.postgres, team.organization_id, 'THIS IS NOT A TOKEN FOR TEAM 3')
        team2 = (await getTeam(infra.postgres, team2Id))!

        jest.mocked(createPrepareEventStep).mockImplementation((...args) => {
            const original = jest.requireActual('~/ingestion/common/steps/event-processing/prepare-event-step')
            return original.createPrepareEventStep(...args)
        })

        jest.mocked(createProcessGroupsStep).mockImplementation((...args) => {
            const original = jest.requireActual('~/ingestion/common/steps/event-processing/process-groups-step')
            return original.createProcessGroupsStep(...args)
        })

        ingester = await createIngestionConsumer(infra)
    })

    afterEach(async () => {
        await ingester.stop()
        await infra.close()
    })

    afterAll(() => {
        jest.useRealTimers()
    })

    describe('general', () => {
        it('should have the correct config', () => {
            expect(ingester['name']).toMatchInlineSnapshot(`"ingestion-consumer-events_plugin_ingestion_test"`)
            expect(ingester['groupId']).toMatchInlineSnapshot(`"events-ingestion-consumer"`)
            expect(ingester['topic']).toMatchInlineSnapshot(`"events_plugin_ingestion_test"`)
        })

        it('should process a standard event', async () => {
            await ingester.handleKafkaBatch(createKafkaMessages([createEvent()]))

            expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
        })

        it('should stop exporting the processed timestamp of revoked partitions', async () => {
            latestOffsetTimestampGauge.reset()
            const messages = createKafkaMessages([createEvent(), createEvent()])
            messages[1].partition = 2
            await ingester.handleKafkaBatch(messages)

            const onPartitionsRevoked = jest.mocked(ingester['kafkaConsumer'].connect).mock.calls[0][1]!
            await onPartitionsRevoked([{ topic: 'test', partition: 1 }])

            const { values } = await latestOffsetTimestampGauge.get()
            expect(values.filter((v) => v.labels.groupId === ingester['groupId'])).toEqual([
                expect.objectContaining({ labels: expect.objectContaining({ topic: 'test', partition: 2 }) }),
            ])
        })

        it('should process a cookieless event', async () => {
            await ingester.handleKafkaBatch(createKafkaMessages([createCookielessEvent()]))

            expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
        })

        it('should drop a cookieless event if the team has cookieless disabled', async () => {
            await infra.postgres.query(
                PostgresUse.COMMON_WRITE,
                `UPDATE posthog_team SET cookieless_server_hash_mode = $1 WHERE id = $2`,
                [CookielessServerHashMode.Disabled, team.id],
                'set cookieless to disabled'
            )
            await ingester.handleKafkaBatch(createKafkaMessages([createCookielessEvent()]))

            expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
            const warningMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
                'clickhouse_ingestion_warnings_test'
            )
            expect(warningMessages).toHaveLength(1)
            expect(warningMessages[0].value).toMatchObject({
                team_id: team.id,
                type: 'cookieless_team_disabled',
            })
            expect(mockProducerObserver.getProducedKafkaMessages()).toHaveLength(1)
        })

        it('should not blend person properties from 2 different cookieless users', async () => {
            const events = [
                createCookielessEvent({
                    ip: '1.2.3.4',
                    properties: {
                        $set: {
                            a: 'a',
                            ab: 'a',
                        },
                    },
                }),
                createCookielessEvent({
                    ip: '5.6.7.8', // different IP address means different user
                    properties: {
                        $set: {
                            b: 'b',
                            ab: 'b',
                        },
                    },
                }),
            ]
            const input = createKafkaMessages(events)
            await ingester.handleKafkaBatch(input)
            const messages = mockProducerObserver.getProducedKafkaMessages()
            const eventsA = messages.filter(
                (m) =>
                    m.topic === 'clickhouse_events_json_test' &&
                    m.value.person_properties &&
                    parseJSON(m.value.person_properties as any)?.a
            )
            const eventsB = messages.filter(
                (m) =>
                    m.topic === 'clickhouse_events_json_test' &&
                    m.value.person_properties &&
                    parseJSON(m.value.person_properties as any)?.b
            )
            expect(eventsA[0].value.person_id).not.toEqual(eventsB[0].value.person_id)
            expect(forSnapshot(eventsA)).toMatchSnapshot()
            expect(forSnapshot(eventsB)).toMatchSnapshot()
        })

        describe('overflow', () => {
            const now = () => DateTime.now().toMillis()
            beforeEach(async () => {
                // Just to make it easy to see what is configured
                expect(infra.config.EVENT_OVERFLOW_BUCKET_CAPACITY).toEqual(1000)
                // Overflow is now gated on explicit mode; the main lane redirects.
                infra.config.INGESTION_OVERFLOW_MODE = 'redirect'
                await ingester.stop()
                ingester = await createIngestionConsumer(infra)
            })

            it('should emit to overflow if token and distinct_id are overflowed', async () => {
                ;(ingester['overflowRedirectService'] as any)['strategies'][0]['limiter'].consume(
                    `${team.api_token}:overflow-distinct-id`,
                    1000,
                    now()
                )
                const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-distinct-id' })])
                await ingester.handleKafkaBatch(overflowMessages)
                expect(
                    mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                ).toHaveLength(0)
                expect(
                    mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
                ).toHaveLength(1)
                expect(
                    forSnapshot(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
                    )
                ).toMatchSnapshot()
            })

            it('does not overflow if it is consuming from the overflow topic', async () => {
                // Overflow lane consumes the overflow topic; it never redirects back.
                infra.config.INGESTION_OVERFLOW_MODE = 'consume'
                const overflowIngester = await createIngestionConsumer(infra, {
                    INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
                })

                // Overflow consumer doesn't have overflowRedirectService (overflowEnabled() returns false)
                // so events are never redirected back to overflow
                expect(overflowIngester['overflowRedirectService']).toBeUndefined()

                const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-distinct-id' })])
                await overflowIngester.handleKafkaBatch(overflowMessages)

                expect(
                    mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
                ).toHaveLength(0)
                expect(
                    mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                ).toHaveLength(1)

                await overflowIngester.stop()
            })

            describe('stateful overflow redirect', () => {
                it('refreshes Redis TTL when processing events in overflow lane', async () => {
                    infra.config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS = 300
                    infra.config.INGESTION_LANE = 'overflow'
                    infra.config.INGESTION_OVERFLOW_MODE = 'consume'

                    // Create overflow lane consumer
                    const overflowIngester = await createIngestionConsumer(infra, {
                        INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
                    })

                    // Verify TTL refresh service is created
                    expect(overflowIngester['overflowLaneTTLRefreshService']).toBeDefined()

                    // Pre-seed Redis with overflow flags (simulating main lane has flagged these keys)
                    const redis = await infra.redisPool.acquire()
                    const redisKey = `@posthog/stateful-overflow/events:${team.api_token}:overflow-user`
                    await redis.set(redisKey, '1', 'EX', 300)

                    // Get initial TTL
                    const initialTTL = await redis.ttl(redisKey)
                    expect(initialTTL).toBeGreaterThan(0)
                    expect(initialTTL).toBeLessThanOrEqual(300)

                    // Process an event in the overflow lane
                    const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'overflow-user' })])
                    await overflowIngester.handleKafkaBatch(overflowMessages)

                    // Verify TTL was refreshed (should be back to ~300 seconds)
                    const refreshedTTL = await redis.ttl(redisKey)
                    expect(refreshedTTL).toBeGreaterThan(0)
                    expect(refreshedTTL).toBeLessThanOrEqual(300)
                    // The TTL should be close to the configured value after refresh
                    expect(refreshedTTL).toBeGreaterThanOrEqual(298) // Allow for some time to pass

                    // Verify the event was still processed normally
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(1)

                    await infra.redisPool.release(redis)
                    await overflowIngester.stop()
                })

                it('does not create keys when refreshing TTL for non-existent keys', async () => {
                    infra.config.INGESTION_STATEFUL_OVERFLOW_REDIS_TTL_SECONDS = 300
                    infra.config.INGESTION_LANE = 'overflow'
                    infra.config.INGESTION_OVERFLOW_MODE = 'consume'

                    // Create overflow lane consumer
                    const overflowIngester = await createIngestionConsumer(infra, {
                        INGESTION_CONSUMER_CONSUME_TOPIC: 'events_plugin_ingestion_overflow_test',
                    })

                    const redis = await infra.redisPool.acquire()
                    const redisKey = `@posthog/stateful-overflow/events:${team.api_token}:new-user`

                    // Verify key doesn't exist initially
                    expect(await redis.exists(redisKey)).toBe(0)

                    // Process an event for a user that wasn't flagged
                    const overflowMessages = createKafkaMessages([createEvent({ distinct_id: 'new-user' })])
                    await overflowIngester.handleKafkaBatch(overflowMessages)

                    // Verify key was NOT created (GETEX doesn't create keys, only refreshes existing ones)
                    expect(await redis.exists(redisKey)).toBe(0)

                    // Verify the event was still processed
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(1)

                    await infra.redisPool.release(redis)
                    await overflowIngester.stop()
                })
            })

            describe('force overflow', () => {
                let ingester: IngestionConsumer

                afterEach(async () => {
                    await ingester.stop()
                })

                it('should force events with matching token:distinct_id to overflow', async () => {
                    infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `${team.api_token}:team1-user`
                    ingester = await createIngestionConsumer(infra)

                    const messages = [
                        createKafkaMessage(createEvent({ distinct_id: 'team1-user' }), team.api_token), // should overflow
                        createKafkaMessage(createEvent({ distinct_id: 'team1-other' }), team.api_token), // should not overflow (different distinct_id)
                        createKafkaMessage(createEvent({ distinct_id: 'team1-user' }), team2.api_token), // should not overflow (different token)
                    ]

                    await ingester.handleKafkaBatch(messages)

                    // Only the matching token:distinct_id event should be routed to overflow
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
                    ).toHaveLength(1)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(2)

                    // Verify the right event went to overflow and the right events were processed normally
                    const overflowMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
                        'events_plugin_ingestion_overflow_test'
                    )
                    const normalMessages =
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')

                    expect(overflowMessages[0].value.distinct_id).toEqual('team1-user')
                    expect(overflowMessages[0].value.token).toEqual(team.api_token)
                    expect(normalMessages.map((m) => m.value.distinct_id).sort()).toEqual(['team1-other', 'team1-user'])

                    // Add snapshot for the overflow messages
                    expect(forSnapshot(overflowMessages)).toMatchSnapshot('force overflow messages')
                })

                it('should handle multiple token:distinct_id pairs in force overflow setting', async () => {
                    infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `${team.api_token}:user1,${team2.api_token}:user2`
                    ingester = await createIngestionConsumer(infra)

                    const messages = [
                        createKafkaMessage(createEvent({ distinct_id: 'user1' }), team.api_token), // should overflow
                        createKafkaMessage(createEvent({ distinct_id: 'other' }), team.api_token), // should not overflow
                        createKafkaMessage(createEvent({ distinct_id: 'user2' }), team2.api_token), // should overflow
                        createKafkaMessage(createEvent({ distinct_id: 'other' }), team2.api_token), // should not overflow
                    ]

                    await ingester.handleKafkaBatch(messages)

                    // Both matching token:distinct_id pairs should be routed to overflow
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_overflow_test')
                    ).toHaveLength(2)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(2)

                    // Verify both matching events went to overflow
                    const overflowMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
                        'events_plugin_ingestion_overflow_test'
                    )
                    const normalMessages =
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')

                    // Sort messages by distinct_id to make the test deterministic
                    const sortedOverflowMessages = [...overflowMessages].sort((a, b) =>
                        String(a.value.distinct_id).localeCompare(String(b.value.distinct_id))
                    )

                    expect(sortedOverflowMessages.map((m) => [m.value.token, m.value.distinct_id])).toEqual([
                        [team.api_token, 'user1'],
                        [team2.api_token, 'user2'],
                    ])
                    expect(normalMessages.map((m) => m.value.distinct_id).sort()).toEqual(['other', 'other'])

                    // Add snapshot for the overflow messages
                    expect(forSnapshot(sortedOverflowMessages)).toMatchSnapshot(
                        'force overflow messages multiple pairs'
                    )
                })

                describe('via headers (preprocessing)', () => {
                    it('forces overflow using headers even if payload token/distinct_id differ', async () => {
                        infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
                        ingester = await createIngestionConsumer(infra)

                        const event = createEvent({ distinct_id: 'not-forced' })
                        const [message] = createKafkaMessages([event])

                        message.headers = [
                            { token: Buffer.from('forced-token') },
                            { distinct_id: Buffer.from('forced-id') },
                        ]

                        await ingester.handleKafkaBatch([message])

                        expect(
                            mockProducerObserver.getProducedKafkaMessagesForTopic(
                                'events_plugin_ingestion_overflow_test'
                            )
                        ).toHaveLength(1)
                        expect(
                            mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                        ).toHaveLength(0)
                    })

                    it('preserves partition locality when not skipping person; drops key when skipping person', async () => {
                        // Not skipping person -> key preserved
                        infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
                        infra.config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID = ''
                        infra.config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY = true
                        ingester = await createIngestionConsumer(infra)

                        const eventA = createEvent({ distinct_id: 'not-forced' })
                        const [messageA] = createKafkaMessages([eventA])
                        const originalKeyA = messageA.key
                        messageA.headers = [
                            { token: Buffer.from('forced-token') },
                            { distinct_id: Buffer.from('forced-id') },
                        ]
                        await ingester.handleKafkaBatch([messageA])
                        const overflowA = mockProducerObserver.getProducedKafkaMessagesForTopic(
                            'events_plugin_ingestion_overflow_test'
                        )
                        expect(overflowA).toHaveLength(1)
                        expect(overflowA[0].key).toEqual(originalKeyA)

                        // Reset produced messages
                        mockProducerObserver.resetKafkaProducer()

                        // Skipping person -> key dropped when preserve locality disabled by config
                        await ingester.stop()
                        infra.config.INGESTION_FORCE_OVERFLOW_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
                        infra.config.SKIP_PERSONS_PROCESSING_BY_TOKEN_DISTINCT_ID = `forced-token:forced-id`
                        infra.config.INGESTION_OVERFLOW_PRESERVE_PARTITION_LOCALITY = false
                        ingester = await createIngestionConsumer(infra)

                        const eventB = createEvent({ distinct_id: 'not-forced' })
                        const [messageB] = createKafkaMessages([eventB])
                        messageB.headers = [
                            { token: Buffer.from('forced-token') },
                            { distinct_id: Buffer.from('forced-id') },
                        ]
                        await ingester.handleKafkaBatch([messageB])
                        const overflowB = mockProducerObserver.getProducedKafkaMessagesForTopic(
                            'events_plugin_ingestion_overflow_test'
                        )
                        expect(overflowB).toHaveLength(1)
                        expect(overflowB[0].key).toBeNull()
                    })
                })
            })
        })
    })

    describe('dropping events', () => {
        it('should DLQ $exception events (wrong consumer)', async () => {
            const messages = createKafkaMessages([
                createEvent({
                    event: '$exception',
                    properties: { $exception_message: 'Test error' },
                }),
            ])

            await ingester.handleKafkaBatch(messages)

            expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
            expect(
                mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_dlq_test')
            ).toHaveLength(1)
        })

        it('should DLQ $$client_ingestion_warning events (wrong consumer)', async () => {
            const messages = createKafkaMessages([
                createEvent({
                    event: '$$client_ingestion_warning',
                    properties: { $$client_ingestion_warning_message: 'test' },
                }),
            ])

            await ingester.handleKafkaBatch(messages)

            expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)
            expect(
                mockProducerObserver.getProducedKafkaMessagesForTopic('events_plugin_ingestion_dlq_test')
            ).toHaveLength(1)
        })

        it('should capture ingestion warning for $groupidentify with too long $group_key', async () => {
            const longKey = 'x'.repeat(401)
            const messages = createKafkaMessages([
                createEvent({
                    event: '$groupidentify',
                    properties: {
                        $group_key: longKey,
                        $group_type: 'company',
                        $group_set: { name: 'Test Company' },
                    },
                }),
            ])

            const { backgroundTask } = await ingester.handleKafkaBatch(messages)
            await backgroundTask

            // Event should be dropped
            expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)

            // Should produce ingestion warning
            const warningMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
                'clickhouse_ingestion_warnings_test'
            )
            expect(warningMessages).toHaveLength(1)
            expect(warningMessages[0].value).toMatchObject({
                team_id: team.id,
                type: 'group_key_too_long',
                details: expect.stringContaining(longKey),
            })
        })

        describe.each(['headers', 'payload'] as const)('via %s', (kind) => {
            const addMessageHeaders = (message: Message, token?: string, distinctId?: string) => {
                if (kind !== 'headers') {
                    return
                }
                message.headers = []

                if (distinctId) {
                    message.headers.push({
                        distinct_id: Buffer.from(distinctId),
                    })
                }
                if (token) {
                    message.headers.push({
                        token: Buffer.from(token),
                    })
                }
            }

            beforeEach(() => {
                jest.spyOn(logger, 'debug')
            })

            describe('with DROP_EVENTS_BY_TOKEN_DISTINCT_ID drops events with matching token:distinct_id when only event keys are listed', () => {
                beforeEach(async () => {
                    infra.config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID = `${team.api_token}:distinct-id-to-ignore,phc_other:distinct-id-to-ignore`
                    ingester = await createIngestionConsumer(infra)
                })
                it('should drop events with matching token and distinct_id', async () => {
                    const messages = createKafkaMessages([
                        createEvent({
                            distinct_id: 'distinct-id-to-ignore',
                        }),
                    ])
                    addMessageHeaders(messages[0], team.api_token, 'distinct-id-to-ignore')
                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(0)
                })

                it('should not drop events for a different team token', async () => {
                    const messages = createKafkaMessages(
                        [
                            createEvent({
                                distinct_id: 'distinct-id-to-ignore',
                            }),
                        ],
                        team2.api_token
                    )
                    addMessageHeaders(messages[0], team2.api_token, 'distinct-id-to-ignore')
                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).not.toHaveLength(0)
                })

                it('should not drop events for a different distinct_id', async () => {
                    const messages = createKafkaMessages([
                        createEvent({
                            distinct_id: 'other-id',
                        }),
                    ])
                    addMessageHeaders(messages[0], team.api_token, 'not-ignored')
                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).not.toHaveLength(0)
                })
            })

            describe('with DROP_EVENTS_BY_TOKEN_DISTINCT_ID drops all team events when only token is listed', () => {
                beforeEach(async () => {
                    const distinct_id_to_drop = 'team1_user_to_drop'
                    infra.config.DROP_EVENTS_BY_TOKEN_DISTINCT_ID = `${team.api_token}:${distinct_id_to_drop},${team2.api_token}`
                    ingester = await createIngestionConsumer(infra)
                })

                it('should still drop events with matching token and distinct_id (event key)', async () => {
                    const distinct_id_to_drop = 'team1_user_to_drop'
                    const messages = createKafkaMessages([
                        createEvent({
                            distinct_id: distinct_id_to_drop,
                        }),
                    ])
                    addMessageHeaders(messages[0], team.api_token, distinct_id_to_drop)
                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(0)
                })

                it('should not drop all events for team with event key listed when distinct_id differs in event', async () => {
                    const unlisted_distinct_id = 'team1_user_NOT_to_drop'
                    const messages = createKafkaMessages([
                        createEvent({
                            distinct_id: unlisted_distinct_id,
                        }),
                    ])
                    addMessageHeaders(messages[0], team.api_token, unlisted_distinct_id)
                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).not.toHaveLength(0)
                })

                it('should drop all events for team with only token listed to be dropped', async () => {
                    const any_distinct_id = 'any_user'
                    const other_distinct_id = 'other_user'

                    const messages = createKafkaMessages(
                        [
                            createEvent({
                                distinct_id: any_distinct_id,
                            }),
                            createEvent({
                                distinct_id: other_distinct_id,
                            }),
                        ],
                        team2.api_token
                    )
                    addMessageHeaders(messages[0], team2.api_token, any_distinct_id)
                    addMessageHeaders(messages[1], team2.api_token, other_distinct_id)

                    await ingester.handleKafkaBatch(messages)
                    expect(
                        mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')
                    ).toHaveLength(0)
                })
            })
        })
    })

    describe('event batching', () => {
        beforeEach(async () => {
            ingester = await createIngestionConsumer(infra)
        })

        it('should validate historical_migration flag based on timestamp age', async () => {
            const ingestionTime = fixedTime // Now = ingestion time
            const oldEventTime = ingestionTime.minus({ hours: 50 }) // Event happened 50 hours ago (>48 hours)
            const recentEventTime = ingestionTime.minus({ hours: 24 }) // Event happened 24 hours ago (<48 hours)

            const events = [
                createEvent({
                    distinct_id: 'user-old-event',
                    event: 'old-event',
                    timestamp: oldEventTime.toISO()!, // Event time in the event payload
                    now: ingestionTime.toISO()!, // Ingestion time in the event payload
                }),
                createEvent({
                    distinct_id: 'user-recent-event',
                    event: 'recent-event',
                    timestamp: recentEventTime.toISO()!, // Event time in the event payload
                    now: ingestionTime.toISO()!, // Ingestion time in the event payload
                }),
            ]

            // Create messages with different event timestamps but same ingestion time (now)
            // The validate-historical-migration step compares: now - timestamp >= 48 hours
            const messages: Message[] = [
                {
                    ...createKafkaMessage(events[0], team.api_token),
                    headers: [
                        { token: Buffer.from(team.api_token) },
                        { distinct_id: Buffer.from('user-old-event') },
                        { timestamp: Buffer.from(oldEventTime.toISO()!) }, // Event time (when it actually happened)
                        { now: Buffer.from(ingestionTime.toISO()!) }, // Ingestion time (when we received it)
                        { historical_migration: Buffer.from('true') },
                    ],
                },
                {
                    ...createKafkaMessage(events[1], team.api_token),
                    headers: [
                        { token: Buffer.from(team.api_token) },
                        { distinct_id: Buffer.from('user-recent-event') },
                        { timestamp: Buffer.from(recentEventTime.toISO()!) }, // Event time (when it actually happened)
                        { now: Buffer.from(ingestionTime.toISO()!) }, // Ingestion time (when we received it)
                        { historical_migration: Buffer.from('true') },
                    ],
                },
            ]

            await ingester.handleKafkaBatch(messages)

            const producedMessages = mockProducerObserver.getProducedKafkaMessages()
            const eventsTopicMessages = producedMessages.filter((m) => m.topic === 'clickhouse_events_json_test')

            expect(eventsTopicMessages).toHaveLength(2)

            // Old event should have historical_migration: true (>48 hours)
            const oldEventMessage = eventsTopicMessages.find((m) => m.value.event === 'old-event')
            expect(oldEventMessage).toBeDefined()
            expect(oldEventMessage?.value.historical_migration).toBe(true)

            // Recent event should not have historical_migration set (<48 hours, defaults to false)
            const recentEventMessage = eventsTopicMessages.find((m) => m.value.event === 'recent-event')
            expect(recentEventMessage).toBeDefined()
            expect(recentEventMessage?.value.historical_migration).toBeUndefined()
        })
    })

    describe('error handling', () => {
        let messages: Message[]

        beforeEach(() => {
            // Simulate some sort of error happening by mocking out the prepare event step
            messages = createKafkaMessages([createEvent()])
            jest.spyOn(logger, 'error').mockImplementation(() => {})
        })

        it('should handle explicitly non retriable errors by sending to DLQ', async () => {
            // Non-retriable errors are converted to DLQ results by the retry wrapper,
            // so the throwing step must be one that carries a retry option.
            const error: any = new Error('test')
            error.isRetriable = false

            // Mock the process groups step (retry-wrapped) to throw the error
            jest.mocked(createProcessGroupsStep).mockImplementation(() => {
                return async function processGroupsStepWrapper() {
                    return Promise.reject(error)
                }
            })

            const ingester = await createIngestionConsumer(infra)
            await ingester.handleKafkaBatch(messages)

            expect(jest.mocked(logger.error)).toHaveBeenCalledWith(
                '🔥',
                'Step process_groups failed',
                expect.any(Object)
            )

            expect(forSnapshot(mockProducerObserver.getProducedKafkaMessages())).toMatchSnapshot()
        })

        it.each([undefined, true])('should throw if isRetriable is set to %s', async (isRetriable) => {
            const error: any = new Error('test')
            error.isRetriable = isRetriable

            // Mock the prepare event step to throw the error
            jest.mocked(createPrepareEventStep).mockImplementation(() => {
                return async function prepareEventStepWrapper() {
                    return Promise.reject(error)
                }
            })

            const ingester = await createIngestionConsumer(infra)
            await expect(ingester.handleKafkaBatch(messages)).rejects.toThrow()
        })

        it('should throw non retriable errors from steps without a retry option', async () => {
            // Steps without a retry option have no error classification: an
            // explicitly non-retriable error crashes the batch instead of DLQing.
            const error: any = new Error('test')
            error.isRetriable = false

            // Mock the prepare event step (no retry option) to throw the error
            jest.mocked(createPrepareEventStep).mockImplementation(() => {
                return async function prepareEventStepWrapper() {
                    return Promise.reject(error)
                }
            })

            const ingester = await createIngestionConsumer(infra)
            await expect(ingester.handleKafkaBatch(messages)).rejects.toThrow()
        })

        it('should emit failures to dead letter queue for non-retriable errors', async () => {
            const error = new Error('Non-retriable processing error')
            const errorAny = error as any
            errorAny.isRetriable = false

            // Mock the process groups step (retry-wrapped) to throw the error
            jest.mocked(createProcessGroupsStep).mockImplementation(() => {
                return async function processGroupsStepWrapper() {
                    return Promise.reject(error)
                }
            })

            const ingester = await createIngestionConsumer(infra)
            await ingester.handleKafkaBatch(messages)

            // Should not produce to clickhouse
            expect(mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_events_json_test')).toHaveLength(0)

            // Should produce to DLQ
            const dlqMessages = mockProducerObserver.getProducedKafkaMessagesForTopic(
                'events_plugin_ingestion_dlq_test'
            )
            expect(dlqMessages).toHaveLength(1)
            expect(dlqMessages[0].value).toEqual(parseJSON(messages[0].value!.toString()))
            expect(dlqMessages[0].headers!['dlq_reason']).toEqual(error.message)
        })
    })

    describe('typical event processing', () => {
        /**
         * NOTE: The majority of these tests should be done in the event pipeline runner but
         * this is a good place to have some high level happy paths
         */

        beforeEach(async () => {
            ingester = await createIngestionConsumer(infra)
        })

        const eventTests: [string, () => PipelineEvent[]][] = [
            ['normal event', () => [createEvent()]],
            [
                '$identify event',
                () => [createEvent({ event: '$identify', properties: { $set: { $email: '[email protected]' } } })],
            ],
            [
                'multiple events',
                () => [
                    createEvent({
                        event: '$pageview',
                        distinct_id: 'anonymous-id-1',
                        properties: { $current_url: 'https://example.com/page1' },
                    }),
                    createEvent({
                        event: '$identify',
                        distinct_id: 'identified-id-1',
                        properties: { $set: { $email: '[email protected]' }, $anonymous_distinct_id: 'anonymous-id-1' },
                    }),
                    createEvent({
                        event: '$pageview',
                        distinct_id: 'identified-id-1',
                        properties: { $current_url: 'https://example.com/page2' },
                    }),
                ],
            ],
            // [
            //     'heatmap event',
            //     () => [
            //         createEvent({
            //             distinct_id: 'distinct-id-1',
            //             event: '$$heatmap',
            //             properties: {
            //                 $heatmap_data: {
            //                     'http://localhost:3000/': [
            //                         {
            //                             x: 1020,
            //                             y: 363,
            //                             target_fixed: false,
            //                             type: 'mousemove',
            //                         },
            //                         {
            //                             x: 634,
            //                             y: 460,
            //                             target_fixed: false,
            //                             type: 'click',
            //                         },
            //                     ],
            //                 },
            //             },
            //         }),
            //     ],
            // ],
            [
                // NOTE: This currently returns as is - for now we keep this broken but once we release the new ingester we should fix this
                'malformed person information',
                () => [
                    createEvent({
                        distinct_id: 'distinct-id-1',
                        properties: { $set: 'INVALID', $unset: [[[['definitel invalid']]]] },
                    }),
                ],
            ],
            ['malformed event', () => [createEvent({ event: '' })]],
            ['event with common distinct_id that gets dropped', () => [createEvent({ distinct_id: 'distinct_id' })]],
            [
                'ai event',
                () => [
                    createEvent({
                        event: '$ai_generation',
                        properties: {
                            $ai_model: 'gpt-4',
                            $ai_provider: 'openai',
                            $ai_input_tokens: 100,
                            $ai_output_tokens: 50,
                        },
                    }),
                ],
            ],
            [
                'person processing off',
                () => [createEvent({ event: '$pageview', properties: { $process_person_profile: false } })],
            ],
            [
                'forced person upgrade',
                () => [
                    createEvent({
                        event: '$pageview',
                        properties: { $process_person_profile: false, $set: { update1: '1' } },
                    }),
                    createEvent({
                        event: '$identify',
                        properties: { $process_person_profile: true, $set: { email: '[email protected]' } },
                    }),
                    // Add an event at least a minute in the future and it should get force upgraded
                    createEvent({
                        event: '$pageview',
                        properties: { $process_person_profile: false, $set: { update2: '2' } },
                        timestamp: DateTime.now().plus({ minutes: 2 }).toISO(),
                    }),
                    // Add a person-full event and ensure all properties are there that should be
                    createEvent({
                        event: '$pageview',
                        properties: { $process_person_profile: true, $set: { update3: '3' } },
                        timestamp: DateTime.now().plus({ minutes: 3 }).toISO(),
                    }),
                    // Snapshot should contain update2 and update3 but not update1
                ],
            ],
            [
                'groups',
                () => [
                    createEvent({
                        event: '$pageview',
                        properties: {
                            $groups: {
                                a: 'group-a',
                                b: 'group-b',
                                c: 'group-c',
                                d: 'group-d',
                                e: 'group-e',
                                f: 'group-f',
                            },
                        },
                    }),
                    createEvent({
                        event: '$groupidentify',
                        properties: {
                            $group_type: 'a',
                            $group_key: 'group-a',
                            $group_set: {
                                id: 'group-a',
                                foo: 'bar',
                            },
                        },
                    }),
                    // This triggers an event but not a groups clickhouse change as the max groups is already hit
                    createEvent({
                        event: '$groupidentify',
                        properties: {
                            $group_type: 'f',
                            $group_key: 'group-f',
                            $group_set: {
                                id: 'group-f',
                                foo: 'bar',
                            },
                        },
                    }),
                ],
            ],
            [
                'person property merging via alias',
                () => {
                    const anonId1 = new UUIDT().toString()
                    const anonId2 = new UUIDT().toString()
                    return [
                        createEvent({
                            distinct_id: anonId1,
                            event: 'custom event',
                            properties: { $set: { k: 'v' } },
                        }),
                        createEvent({
                            distinct_id: anonId2,
                            event: 'custom event',
                            properties: { $set: { j: 'w' } },
                        }),
                        // final event should have k, j, l
                        createEvent({
                            distinct_id: anonId2,
                            event: '$create_alias',
                            properties: { alias: anonId1, $set: { l: 'x' } },
                        }),
                    ]
                },
            ],
        ]

        it.each(eventTests)('%s', async (_, createEvents) => {
            const messages = createKafkaMessages(createEvents())
            await ingester.handleKafkaBatch(messages)

            // Tricky due to some parallel processing race conditions order isn't deterministic
            // So we sort by specific properties to make it deterministic
            const sortingKey = (message: DecodedKafkaMessage) => {
                const value = message.value
                return `${value.topic}:${value.team_id}:${value.distinct_id}:${value.properties}`
            }

            const sortedMessages = mockProducerObserver
                .getProducedKafkaMessages()
                .sort((a, b) => sortingKey(a).localeCompare(sortingKey(b)))

            expect(forSnapshot(sortedMessages)).toMatchSnapshot()
        })
    })

    describe('transformations', () => {
        let transformationFunction: HogFunctionType
        const TRANSFORMATION_TEST_TIMEOUT = 30000

        beforeAll(() => {
            jest.setTimeout(TRANSFORMATION_TEST_TIMEOUT)
        })

        afterAll(() => {
            jest.setTimeout(DEFAULT_TEST_TIMEOUT)
        })

        const insertHogFunction = async (hogFunction: Partial<HogFunctionType>) => {
            const { hog, bytecode, name } = hogFunction
            const item = await _insertHogFunction(infra.postgres, team.id, {
                hog,
                bytecode,
                name: name || 'Test Function',
                type: 'transformation',
            })
            return item
        }

        beforeEach(async () => {
            // Create a transformation function using the geoip template as an example
            const hogByteCode = await compileHog(geoipTemplate.code)
            transformationFunction = await insertHogFunction({
                name: 'GeoIP Transformation',
                hog: geoipTemplate.code,
                bytecode: hogByteCode,
            })

            ingester = await createIngestionConsumer(infra)
        })

        it(
            'should invoke transformation for matching team with error case',
            async () => {
                // make the geoip lookup fail
                const event = createEvent({
                    ip: '256.256.256.256',
                    properties: { $ip: '256.256.256.256' },
                })
                const messages = createKafkaMessages([event])

                const result = await ingester.handleKafkaBatch(messages)
                await result.backgroundTask

                // Verify metrics were published
                const metricsMessages =
                    mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
                expect(metricsMessages).toEqual([
                    {
                        headers: undefined,
                        key: null,
                        topic: 'clickhouse_app_metrics2_test',
                        value: {
                            app_source: 'hog_function',
                            app_source_id: transformationFunction.id,
                            count: 1,
                            instance_id: '',
                            metric_kind: 'success',
                            metric_name: 'succeeded',
                            team_id: team.id,
                            timestamp: '2025-01-01 00:00:00.000',
                        },
                    },
                ])

                // Verify log entries were published
                const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')
                expect(logMessages).toEqual([
                    {
                        key: null,
                        topic: 'log_entries_test',
                        value: {
                            instance_id: expect.any(String),
                            level: 'info',
                            log_source: 'hog_function',
                            log_source_id: transformationFunction.id,
                            message: 'geoip lookup failed for ip, 256.256.256.256',
                            team_id: team.id,
                            timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
                        },
                    },
                    {
                        key: null,
                        topic: 'log_entries_test',
                        value: {
                            instance_id: expect.any(String),
                            level: 'debug',
                            log_source: 'hog_function',
                            log_source_id: transformationFunction.id,
                            message: expect.stringMatching(/^Function completed in/),
                            team_id: team.id,
                            timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
                        },
                    },
                ])
            },
            TRANSFORMATION_TEST_TIMEOUT
        )

        it(
            'should invoke transformation for matching team with success case',
            async () => {
                const event = createEvent({
                    ip: '89.160.20.129',
                    properties: { $ip: '89.160.20.129' },
                })
                const messages = createKafkaMessages([event])

                const result = await ingester.handleKafkaBatch(messages)
                await result.backgroundTask

                // Verify metrics were published
                const metricsMessages =
                    mockProducerObserver.getProducedKafkaMessagesForTopic('clickhouse_app_metrics2_test')
                expect(metricsMessages).toEqual([
                    {
                        headers: undefined,
                        key: null,
                        topic: 'clickhouse_app_metrics2_test',
                        value: {
                            app_source: 'hog_function',
                            app_source_id: transformationFunction.id,
                            count: 1,
                            instance_id: '',
                            metric_kind: 'success',
                            metric_name: 'succeeded',
                            team_id: team.id,
                            timestamp: '2025-01-01 00:00:00.000',
                        },
                    },
                ])

                // Verify log entries were published
                const logMessages = mockProducerObserver.getProducedKafkaMessagesForTopic('log_entries_test')
                expect(logMessages).toEqual([
                    {
                        key: null,
                        topic: 'log_entries_test',
                        value: {
                            instance_id: expect.any(String),
                            level: 'info',
                            log_source: 'hog_function',
                            log_source_id: transformationFunction.id,
                            message: expect.stringContaining('geoip location data for ip:'),
                            team_id: team.id,
                            timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
                        },
                    },
                    {
                        key: null,
                        topic: 'log_entries_test',
                        value: {
                            instance_id: expect.any(String),
                            level: 'debug',
                            log_source: 'hog_function',
                            log_source_id: transformationFunction.id,
                            message: expect.stringMatching(/^Function completed in/),
                            team_id: team.id,
                            timestamp: expect.stringMatching(/2025-01-01 00:00:00\.\d{3}/),
                        },
                    },
                ])
            },
            TRANSFORMATION_TEST_TIMEOUT
        )
    })

    describe('store flush behavior', () => {
        it('should flush stores exactly once per batch regardless of number of events', async () => {
            const flushSpy = jest.spyOn(ingester['personsStore'], 'flush')
            const groupFlushSpy = jest.spyOn(ingester['groupStore'], 'flush')

            const events: PipelineEvent[] = [
                {
                    distinct_id: 'user1',
                    event: '$pageview',
                    properties: {},
                    timestamp: '2025-01-01 00:00:00',
                    team_id: team.id,
                    now: '2025-01-01 00:00:00',
                    uuid: new UUIDT().toString(),
                    ip: '127.0.0.1',
                    site_url: 'https://example.com',
                },
                {
                    distinct_id: 'user2',
                    event: '$pageview',
                    properties: {},
                    timestamp: '2025-01-01 00:00:01',
                    team_id: team.id,
                    now: '2025-01-01 00:00:01',
                    uuid: new UUIDT().toString(),
                    ip: '127.0.0.1',
                    site_url: 'https://example.com',
                },
                {
                    distinct_id: 'user3',
                    event: '$pageview',
                    properties: {},
                    timestamp: '2025-01-01 00:00:02',
                    team_id: team.id,
                    now: '2025-01-01 00:00:02',
                    uuid: new UUIDT().toString(),
                    ip: '127.0.0.1',
                    site_url: 'https://example.com',
                },
            ]

            const messages = createKafkaMessages(events)

            await ingester.handleKafkaBatch(messages)

            expect(flushSpy).toHaveBeenCalledTimes(1)
            expect(groupFlushSpy).toHaveBeenCalledTimes(1)

            flushSpy.mockRestore()
            groupFlushSpy.mockRestore()
        })

        it('should flush persons and group stores for each batch', async () => {
            const callOrder: string[] = []

            const flushSpy = jest.spyOn(ingester['personsStore'], 'flush').mockImplementation(() => {
                callOrder.push('personsFlush')
                return Promise.resolve([])
            })
            const groupFlushSpy = jest.spyOn(ingester['groupStore'], 'flush').mockImplementation(() => {
                callOrder.push('groupFlush')
                return Promise.resolve([])
            })

            const events: PipelineEvent[] = [
                {
                    distinct_id: 'user1',
                    event: '$pageview',
                    properties: {},
                    timestamp: '2025-01-01 00:00:00',
                    team_id: team.id,
                    now: '2025-01-01 00:00:00',
                    uuid: new UUIDT().toString(),
                    ip: '127.0.0.1',
                    site_url: 'https://example.com',
                },
            ]

            await ingester.handleKafkaBatch(createKafkaMessages(events))

            expect(callOrder).toContain('personsFlush')
            expect(callOrder).toContain('groupFlush')

            flushSpy.mockRestore()
            groupFlushSpy.mockRestore()
        })
    })

    describe('multi-batch cache eviction and ordering', () => {
        it('person properties from batch 1 are visible in batch 2', async () => {
            await ingester.handleKafkaBatch(
                createKafkaMessages([createEvent({ properties: { $set: { name: 'Alice' } } })])
            )

            const personsAfterBatch1 = await fetchPostgresPersons(infra.postgres, team.id)
            expect(personsAfterBatch1).toHaveLength(1)
            expect(personsAfterBatch1[0].properties).toMatchObject({ name: 'Alice' })

            await ingester.handleKafkaBatch(
                createKafkaMessages([createEvent({ properties: { $set: { email: '[email protected]' } } })])
            )

            const personsAfterBatch2 = await fetchPostgresPersons(infra.postgres, team.id)
            expect(personsAfterBatch2).toHaveLength(1)
            expect(personsAfterBatch2[0].properties).toMatchObject({
                name: 'Alice',
                email: '[email protected]',
            })
        })

        it('cache entries are evicted after each batch completes', async () => {
            const personsStore = ingester['personsStore'] as BatchWritingPersonsStore

            await ingester.handleKafkaBatch(createKafkaMessages([createEvent()]))

            const batchDistinctKeys = (personsStore as any)['batchDistinctKeys'] as Map<number, Set<string>>
            expect(batchDistinctKeys.size).toBe(0)
            expect(personsStore.getCheckCache().size).toBe(0)
        })

        it('cache does not grow unboundedly across many sequential batches', async () => {
            const personsStore = ingester['personsStore'] as BatchWritingPersonsStore

            for (let i = 0; i < 5; i++) {
                await ingester.handleKafkaBatch(createKafkaMessages([createEvent({ distinct_id: `user-${i}` })]))
            }

            const batchDistinctKeys = (personsStore as any)['batchDistinctKeys'] as Map<number, Set<string>>
            expect(batchDistinctKeys.size).toBe(0)
            expect(personsStore.getCheckCache().size).toBe(0)
        })
    })

    describe('stop()', () => {
        it('calls shutdown() on persons and group stores', async () => {
            const personShutdownSpy = jest.spyOn(ingester['personsStore'], 'shutdown')
            const groupShutdownSpy = jest.spyOn(ingester['groupStore'], 'shutdown')

            await ingester.stop()

            expect(personShutdownSpy).toHaveBeenCalledTimes(1)
            expect(groupShutdownSpy).toHaveBeenCalledTimes(1)
        })
    })
})