Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

53.6 KB
Raw Blame
import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'

import { ReadOnlyGroupTypeManager } from '~/common/groups/readonly-group-type-manager'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { TophogOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { COOKIELESS_SENTINEL_VALUE } from '~/common/persons/person-utils'
import { PersonReadRepository } from '~/common/persons/repositories/person-repository'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { EventIngestionRestrictionManager, RestrictionType } from '~/common/utils/event-ingestion-restrictions'
import { parseJSON } from '~/common/utils/json-parse'
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { TeamManager } from '~/common/utils/team-manager'
import { UUIDT } from '~/common/utils/utils'
import { CookielessManager } from '~/ingestion/common/cookieless/cookieless-manager'
import { EventFilterManager } from '~/ingestion/common/event-filters'
import { cond } from '~/ingestion/common/event-filters/test-helpers'
import { OverflowRedirectService } from '~/ingestion/common/overflow-redirect/overflow-redirect-service'
import { TopHogRegistry } from '~/ingestion/framework/extensions/tophog'
import { createOkContext } from '~/ingestion/framework/helpers'
import { ok } from '~/ingestion/framework/results'
import { TopHog } from '~/ingestion/framework/tophog'
import { createTestTeam } from '~/tests/helpers/team'
import { InternalPerson } from '~/types'

import { CymbalClient } from './cymbal/client'
import { CymbalResponse } from './cymbal/types'
import {
    ErrorTrackingHogTransformer,
    ErrorTrackingPipeline,
    ErrorTrackingPipelineConfig,
    createErrorTrackingPipeline,
} from './error-tracking-pipeline'

/** Drives a batch through the pipeline the way the common consumer's batch handler does. */
async function runErrorTrackingPipeline(pipeline: ErrorTrackingPipeline, messages: Message[]): Promise<void> {
    const batch = messages.map((message) => createOkContext({ message }, { message }))
    const feedResult = await pipeline.feed(batch, {})
    if (!feedResult.ok) {
        throw new Error(`Pipeline rejected batch: ${feedResult.reason}`)
    }
    while ((await pipeline.next()) !== null) {
        // Drain all results
    }
}

// Skip retry sleeps so tests run instantly
jest.mock('~/common/utils/utils', () => ({
    ...jest.requireActual('~/common/utils/utils'),
    sleep: jest.fn().mockResolvedValue(undefined),
}))

// Suppress logger output during tests
jest.mock('~/common/utils/logger', () => ({
    logger: {
        debug: jest.fn(),
        info: jest.fn(),
        warn: jest.fn(),
        error: jest.fn(),
    },
}))

describe('ErrorTrackingPipeline', () => {
    let mockKafkaProducer: jest.Mocked<KafkaProducerWrapper>
    let mockTeamManager: jest.Mocked<TeamManager>
    let mockPersonRepository: jest.Mocked<PersonReadRepository>
    let mockHogTransformer: jest.Mocked<ErrorTrackingHogTransformer>
    let mockCymbalClient: jest.Mocked<CymbalClient>
    let mockGroupTypeManager: jest.Mocked<ReadOnlyGroupTypeManager>
    let mockEventIngestionRestrictionManager: jest.Mocked<EventIngestionRestrictionManager>
    let mockEventFilterManager: jest.Mocked<EventFilterManager>
    let mockCookielessManager: jest.Mocked<CookielessManager>
    let promiseScheduler: PromiseScheduler
    let pipelineConfig: ErrorTrackingPipelineConfig

    const team = createTestTeam({ id: 123, api_token: 'test-token-123' })

    const createTestPerson = (
        overrides: Partial<InternalPerson> & { distinct_id?: string } = {}
    ): InternalPerson & { distinct_id: string } => ({
        id: '1',
        uuid: 'person-uuid-123',
        team_id: 123,
        distinct_id: 'user-123', // Default matches createKafkaMessage default
        properties: { email: '[email protected]', name: 'Test User' },
        is_user_id: null,
        is_identified: true,
        created_at: DateTime.utc(2024, 1, 1),
        version: 1,
        last_seen_at: null,
        properties_last_updated_at: {},
        properties_last_operation: null,
        ...overrides,
    })

    const createCymbalResponse = (overrides: Partial<CymbalResponse> = {}): CymbalResponse => ({
        uuid: 'test-uuid',
        event: '$exception',
        team_id: 1,
        timestamp: '2024-01-01T00:00:00Z',
        properties: {
            $exception_list: [{ type: 'Error', value: 'Test error' }],
            $exception_fingerprint: 'test-fingerprint',
            $exception_issue_id: 'test-issue-id',
        },
        ...overrides,
    })

    /**
     * Creates a mock Kafka message with proper headers and body structure.
     */
    const createKafkaMessage = (options: {
        token?: string
        distinctId?: string
        eventUuid?: string
        timestamp?: string
        ip?: string
        event?: string
        properties?: Record<string, any>
    }): Message => {
        const {
            token = 'test-token-123',
            distinctId = 'user-123',
            eventUuid = new UUIDT().toString(),
            timestamp = '2024-01-01T00:00:00Z',
            ip = '1.2.3.4',
            event = '$exception',
            properties = {},
        } = options

        const eventData = {
            event,
            distinct_id: distinctId,
            uuid: eventUuid,
            timestamp,
            ip,
            properties: {
                $exception_list: [{ type: 'Error', value: 'Test error' }],
                ...properties,
            },
        }

        const messageBody = {
            token,
            data: JSON.stringify(eventData),
            ...eventData,
        }

        return {
            value: Buffer.from(JSON.stringify(messageBody)),
            headers: [
                { token: Buffer.from(token) },
                { distinct_id: Buffer.from(distinctId) },
                { uuid: Buffer.from(eventUuid) },
                { timestamp: Buffer.from(timestamp) },
            ],
            topic: 'error_tracking_events',
            partition: 0,
            offset: 0,
            size: 0,
            key: Buffer.from(`${token}:${distinctId}`),
        } as Message
    }

    /**
     * Helper to get events produced to a specific topic.
     */
    const getProducedMessagesForTopic = (topic: string): any[] => {
        return mockKafkaProducer.produce.mock.calls
            .filter((call) => call[0].topic === topic)
            .map((call) => parseJSON(call[0].value!.toString()))
    }

    /**
     * Helper to get events that were produced to the output topic.
     */
    const getProducedEvents = (): any[] => {
        return getProducedMessagesForTopic('clickhouse_events_json_test')
    }

    /**
     * Helper to get events that were sent to the DLQ.
     */
    const getDLQMessages = (): any[] => {
        return getProducedMessagesForTopic('error_tracking_dlq')
    }

    /**
     * Helper to get events that were redirected to overflow.
     */
    const getOverflowMessages = (): any[] => {
        return getProducedMessagesForTopic('error_tracking_overflow')
    }

    const getQueuedMessages = (topicFragment: string): any[] => {
        return mockKafkaProducer.queueMessages.mock.calls
            .filter((call) => {
                const arg = call[0]
                const topicMessages = Array.isArray(arg) ? arg : [arg]
                return topicMessages.some((tm: any) => tm.topic?.includes(topicFragment))
            })
            .flatMap((call) => {
                const arg = call[0]
                const topicMessages = Array.isArray(arg) ? arg : [arg]
                return topicMessages
                    .filter((tm: any) => tm.topic?.includes(topicFragment))
                    .flatMap((tm: any) => tm.messages.map((m: { value: string }) => parseJSON(m.value)))
            })
    }

    const getIngestionWarnings = (): any[] => getQueuedMessages('ingestion_warnings')

    const getAppMetrics = (): any[] => getQueuedMessages('app_metrics')

    /**
     * Creates a Cymbal response that includes enriched properties.
     * In real usage, Cymbal receives the enriched event and returns it with
     * fingerprint/issue_id added. This helper simulates that behavior.
     */
    const createCymbalResponseWithEnrichedProperties = (
        enrichedProperties: Record<string, any>,
        overrides: Partial<CymbalResponse> = {}
    ): CymbalResponse => ({
        uuid: 'test-uuid',
        event: '$exception',
        team_id: 1,
        timestamp: '2024-01-01T00:00:00Z',
        properties: {
            ...enrichedProperties,
            $exception_fingerprint: 'test-fingerprint',
            $exception_issue_id: 'test-issue-id',
        },
        ...overrides,
    })

    beforeEach(() => {
        mockKafkaProducer = {
            produce: jest.fn().mockResolvedValue(undefined),
            queueMessages: jest.fn().mockReturnValue(Promise.resolve()),
            flush: jest.fn().mockResolvedValue(undefined),
        } as unknown as jest.Mocked<KafkaProducerWrapper>

        mockTeamManager = {
            getTeamByToken: jest.fn().mockResolvedValue(team),
            getTeamsByTokens: jest.fn().mockResolvedValue({}),
            getTeam: jest.fn().mockResolvedValue(team),
        } as unknown as jest.Mocked<TeamManager>

        mockPersonRepository = {
            fetchPerson: jest.fn(),
            fetchPersonsByDistinctIds: jest.fn().mockResolvedValue([]),
            fetchPersonsByPersonIds: jest.fn(),
            fetchDistinctIdsForPersons: jest.fn().mockResolvedValue({}),
            createPerson: jest.fn(),
            updatePerson: jest.fn(),
            handleOversizedPersonProperties: jest.fn(),
            updatePersonAssertVersion: jest.fn(),
            updatePersonsBatch: jest.fn(),
            deletePerson: jest.fn(),
            addDistinctId: jest.fn(),
            personPropertiesSize: jest.fn(),
            updateCohortsAndFeatureFlagsForMerge: jest.fn(),
            inTransaction: jest.fn(),
        } as unknown as jest.Mocked<PersonReadRepository>

        // HogTransformer mock that passes through events unchanged by default
        mockHogTransformer = {
            transformEventAndProduceMessages: jest
                .fn()
                .mockImplementation((event) => Promise.resolve({ event, invocationResults: [] })),
            processInvocationResults: jest.fn().mockResolvedValue(undefined),
            prefetchHogFunctionsForTeams: jest.fn().mockResolvedValue(undefined),
        }

        mockCymbalClient = {
            processExceptions: jest.fn(),
            healthCheck: jest.fn(),
        } as unknown as jest.Mocked<CymbalClient>

        mockGroupTypeManager = {
            fetchGroupTypes: jest.fn().mockResolvedValue({}),
            fetchGroupTypeIndex: jest.fn(),
            insertGroupType: jest.fn(),
        } as unknown as jest.Mocked<ReadOnlyGroupTypeManager>

        mockEventIngestionRestrictionManager = {
            getAppliedRestrictions: jest.fn().mockReturnValue(new Set()),
            forceRefresh: jest.fn(),
        } as unknown as jest.Mocked<EventIngestionRestrictionManager>

        mockEventFilterManager = {
            getFilter: jest.fn().mockReturnValue(null),
            dropsAllEvents: jest.fn().mockReturnValue(false),
        } as unknown as jest.Mocked<EventFilterManager>

        // Passthrough: non-cookieless events round-trip unchanged. The apply step
        // extracts `.value.event` from each result and reuses the original input
        // wrapper, so returning `ok(input)` is enough to leave events untouched.
        mockCookielessManager = {
            doBatch: jest.fn().mockImplementation((events: any[]) => Promise.resolve(events.map((e) => ok(e)))),
            shutdown: jest.fn(),
        } as unknown as jest.Mocked<CookielessManager>

        promiseScheduler = new PromiseScheduler()

        // Mock TopHog registry that returns no-op recorders
        const mockRecorder = { record: jest.fn() }
        const mockTopHog: TopHogRegistry = {
            registerSum: () => mockRecorder,
            registerMax: () => mockRecorder,
            registerAverage: () => mockRecorder,
        }

        pipelineConfig = {
            outputs: new IngestionOutputs({
                events: new SingleIngestionOutput('events', 'clickhouse_events_json_test', mockKafkaProducer, 'test'),
                ingestion_warnings: new SingleIngestionOutput(
                    'ingestion_warnings',
                    'clickhouse_ingestion_warnings_test',
                    mockKafkaProducer,
                    'test'
                ),
                dlq: new SingleIngestionOutput('dlq', 'error_tracking_dlq', mockKafkaProducer, 'test'),
                overflow: new SingleIngestionOutput('overflow', 'error_tracking_overflow', mockKafkaProducer, 'test'),
                tophog: new SingleIngestionOutput('tophog', 'clickhouse_tophog_test', mockKafkaProducer, 'test'),
                app_metrics: new SingleIngestionOutput(
                    'app_metrics',
                    'clickhouse_app_metrics2_test',
                    mockKafkaProducer,
                    'test'
                ),
            }),
            promiseScheduler,
            teamManager: mockTeamManager,
            teamsPrefetchEnabled: true,
            hogFunctionsPrefetchEnabled: true,
            personRepository: mockPersonRepository,
            hogTransformer: mockHogTransformer,
            cymbalClient: mockCymbalClient,
            groupTypeManager: mockGroupTypeManager,
            cookielessManager: mockCookielessManager,
            eventIngestionRestrictionManager: mockEventIngestionRestrictionManager,
            eventFilterManager: mockEventFilterManager,
            overflowMode: 'disabled',
            preservePartitionLocality: false,
            topHog: mockTopHog,
            createEventUsageBatch: () => new UsageRecordBatch(null, { unit: 'events', isTeamEnabled: () => false }),
        }
    })

    describe('createErrorTrackingPipeline', () => {
        it('creates a pipeline successfully', () => {
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            expect(pipeline).toBeDefined()
            expect(typeof pipeline.feed).toBe('function')
            expect(typeof pipeline.next).toBe('function')
        })
    })

    describe('runErrorTrackingPipeline', () => {
        it('processes a single event through the full pipeline and emits to Kafka', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])

            // Cymbal receives raw properties (no GeoIP yet) and adds fingerprint/issue_id
            const cymbalResponse = createCymbalResponseWithEnrichedProperties({
                $exception_list: [{ type: 'Error', value: 'Test error' }],
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            // Hog transformations run AFTER Cymbal and add GeoIP properties
            mockHogTransformer.transformEventAndProduceMessages.mockImplementation((event) =>
                Promise.resolve({
                    event: {
                        ...event,
                        properties: {
                            ...event.properties,
                            $geoip_country_code: 'US',
                            $geoip_city_name: 'San Francisco',
                            $geoip_subdivision_1_code: 'CA',
                            $geoip_subdivision_1_name: 'California',
                            $geoip_latitude: 37.7749,
                            $geoip_longitude: -122.4194,
                        },
                    },
                    invocationResults: [],
                })
            )

            const message = createKafkaMessage({})
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Verify Hog transformations were run and their invocation results drained once per batch
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)
            expect(mockHogTransformer.processInvocationResults).toHaveBeenCalledTimes(1)

            // Verify event was emitted to Kafka
            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)

            const emittedEvent = producedEvents[0]
            expect(emittedEvent.event).toBe('$exception')
            expect(emittedEvent.team_id).toBe(123)

            // Verify person fields are on the top-level event (not in properties)
            expect(emittedEvent.person_id).toBe('person-uuid-123')
            expect(parseJSON(emittedEvent.person_properties)).toEqual({
                email: '[email protected]',
                name: 'Test User',
            })

            // Verify properties contain GeoIP (from Hog transformations) and Cymbal data
            const props = parseJSON(emittedEvent.properties)
            expect(props.$geoip_country_code).toBe('US')
            expect(props.$geoip_city_name).toBe('San Francisco')
            expect(props.$exception_fingerprint).toBe('test-fingerprint')
        })

        it('processes multiple events in a batch', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined) // No person

            const uuid1 = new UUIDT().toString()
            const uuid2 = new UUIDT().toString()

            const cymbalResponses = [
                createCymbalResponse({
                    uuid: uuid1,
                    properties: { $exception_fingerprint: 'fp-1', $exception_issue_id: 'issue-1' },
                }),
                createCymbalResponse({
                    uuid: uuid2,
                    properties: { $exception_fingerprint: 'fp-2', $exception_issue_id: 'issue-2' },
                }),
            ]
            mockCymbalClient.processExceptions.mockResolvedValue(cymbalResponses)

            const messages = [
                createKafkaMessage({ distinctId: 'user-1', eventUuid: uuid1 }),
                createKafkaMessage({ distinctId: 'user-2', eventUuid: uuid2 }),
            ]

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, messages)

            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(1)

            // Verify batch was sent to Cymbal
            const cymbalCall = mockCymbalClient.processExceptions.mock.calls[0][0]
            expect(cymbalCall).toHaveLength(2)

            // Verify Hog transformations were run for each event
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(2)

            // Verify both events were emitted
            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(2)
        })

        it('passes Kafka message byte size to Cymbal for batch chunking', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            const cymbalResponse = createCymbalResponseWithEnrichedProperties({
                $exception_list: [{ type: 'Error', value: 'Test error' }],
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            const message = createKafkaMessage({})
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            const cymbalItems = mockCymbalClient.processExceptions.mock.calls[0][0]
            expect(cymbalItems).toHaveLength(1)
            expect(cymbalItems[0].estimatedSize).toBe(message.value!.length)
        })

        it('handles events with group types', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Mock fetchGroupTypes to return the group type to index mapping
            mockGroupTypeManager.fetchGroupTypes.mockResolvedValue({
                company: 0,
                project: 1,
            })

            // Cymbal receives enriched properties (including $group_*) and returns them
            const cymbalResponse = createCymbalResponseWithEnrichedProperties({
                $exception_list: [{ type: 'Error', value: 'Test error' }],
                $groups: { company: 'acme-corp', project: 'project-123' },
                $group_0: 'acme-corp',
                $group_1: 'project-123',
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            const message = createKafkaMessage({
                properties: {
                    $groups: {
                        company: 'acme-corp',
                        project: 'project-123',
                    },
                },
            })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)

            const props = parseJSON(producedEvents[0].properties)
            expect(props.$group_0).toBe('acme-corp')
            expect(props.$group_1).toBe('project-123')
        })

        it('suppresses events when Cymbal returns null response', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Cymbal returns null for suppressed events
            mockCymbalClient.processExceptions.mockResolvedValue([null])

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Suppressed events are dropped before Hog transformations run
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()

            // Suppressed events are dropped, nothing emitted
            expect(getProducedEvents()).toHaveLength(0)
        })

        it('skips processing for empty input', async () => {
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [])

            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
        })

        it('drops events with unknown team token', async () => {
            // Note: Invalid tokens are dropped silently (consistent with analytics pipeline).
            // This differs from billing restrictions which send to DLQ.
            mockTeamManager.getTeamByToken.mockResolvedValue(null)

            const message = createKafkaMessage({ token: 'unknown-token' })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Events with unknown tokens are dropped before Cymbal or Hog transformations
            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()

            // Event should not be produced anywhere (dropped silently)
            expect(getProducedEvents()).toHaveLength(0)
            expect(getDLQMessages()).toHaveLength(0)
            expect(getOverflowMessages()).toHaveLength(0)
        })

        it('sends events to DLQ when REDIRECT_TO_DLQ restriction is set', async () => {
            mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(
                new Set([RestrictionType.REDIRECT_TO_DLQ])
            )

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Events redirected to DLQ skip processing entirely
            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()

            // Event should not be produced to output topic
            expect(getProducedEvents()).toHaveLength(0)

            // Event should be sent to DLQ
            const dlqMessages = getDLQMessages()
            expect(dlqMessages).toHaveLength(1)
            expect(dlqMessages[0].distinct_id).toBe('user-123')
        })

        it('drops events silently when DROP_EVENT restriction is set', async () => {
            mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(
                new Set([RestrictionType.DROP_EVENT])
            )

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Dropped events skip processing entirely
            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()

            // Event should not be produced anywhere
            expect(getProducedEvents()).toHaveLength(0)
            expect(getDLQMessages()).toHaveLength(0)
            expect(getOverflowMessages()).toHaveLength(0)
        })

        it('drops events matching a live event filter before Cymbal and records the drop', async () => {
            mockEventFilterManager.getFilter.mockReturnValue({
                id: 'filter-1',
                team_id: team.id,
                mode: 'live',
                filter_tree: cond('distinct_id', 'exact', 'user-123'),
            })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])

            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(getProducedEvents()).toHaveLength(0)
            expect(getAppMetrics()).toEqual([
                expect.objectContaining({
                    team_id: team.id,
                    app_source: 'event_filter',
                    app_source_id: 'filter-1',
                    metric_name: 'dropped',
                    count: 1,
                }),
            ])
        })

        it('redirects events to overflow when FORCE_OVERFLOW restriction is set', async () => {
            mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(
                new Set([RestrictionType.FORCE_OVERFLOW])
            )

            const message = createKafkaMessage({})

            // Enable overflow for this test
            const configWithOverflow: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                overflowMode: 'redirect',
            }

            const pipeline = createErrorTrackingPipeline(configWithOverflow)
            await runErrorTrackingPipeline(pipeline, [message])

            // Events redirected to overflow skip processing entirely
            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()

            // Event should not be produced to output topic
            expect(getProducedEvents()).toHaveLength(0)

            // Event should be redirected to overflow topic
            const overflowMessages = getOverflowMessages()
            expect(overflowMessages).toHaveLength(1)
            expect(overflowMessages[0].distinct_id).toBe('user-123')
        })

        it('processes events normally when FORCE_OVERFLOW is set but overflow is disabled', async () => {
            // When overflow is disabled, FORCE_OVERFLOW restriction is ignored and
            // events process normally. This allows graceful degradation.
            mockEventIngestionRestrictionManager.getAppliedRestrictions.mockReturnValue(
                new Set([RestrictionType.FORCE_OVERFLOW])
            )
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const message = createKafkaMessage({})

            // Overflow is disabled by default in pipelineConfig
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Event should be processed normally including Hog transformations
            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(1)
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)
            expect(getProducedEvents()).toHaveLength(1)
            expect(getOverflowMessages()).toHaveLength(0)
        })

        it('propagates database errors from person lookup', async () => {
            mockPersonRepository.fetchPersonsByDistinctIds.mockRejectedValue(new Error('Database error'))
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)

            // Database errors should propagate up - they indicate infrastructure issues
            await expect(runErrorTrackingPipeline(pipeline, [message])).rejects.toThrow('Database error')

            // Cymbal was called but Hog transformations weren't reached due to person lookup failure
            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(1)
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()
        })

        it('retries Cymbal errors and propagates after exhausting retries', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Create retriable error (default errors are treated as retriable)
            const retriableError = new Error('Cymbal unavailable')
            ;(retriableError as any).isRetriable = true
            mockCymbalClient.processExceptions.mockRejectedValue(retriableError)

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)

            // Cymbal errors propagate once the retries are exhausted, so Kafka
            // doesn't commit and retries the batch
            await expect(runErrorTrackingPipeline(pipeline, [message])).rejects.toThrow('Cymbal unavailable')

            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(5)
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()
        })

        it('retries Cymbal errors and succeeds on subsequent attempts', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // First two calls fail with retriable error, third call succeeds
            const retriableError = new Error('Cymbal temporarily unavailable')
            ;(retriableError as any).isRetriable = true

            mockCymbalClient.processExceptions
                .mockRejectedValueOnce(retriableError)
                .mockRejectedValueOnce(retriableError)
                .mockResolvedValueOnce([createCymbalResponse()])

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Cymbal was called 3 times before succeeding
            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(3)
            // Processing continued after Cymbal succeeded
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)
            // Event was emitted
            expect(getProducedEvents()).toHaveLength(1)
        })

        it('sends events to DLQ on non-retriable Cymbal errors', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Non-retriable error (e.g., 400 Bad Request from Cymbal)
            const nonRetriableError = new Error('Bad request - invalid event format')
            ;(nonRetriableError as any).isRetriable = false
            mockCymbalClient.processExceptions.mockRejectedValue(nonRetriableError)

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Non-retriable errors should not be retried
            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(1)
            // Processing should not continue past Cymbal
            expect(mockHogTransformer.transformEventAndProduceMessages).not.toHaveBeenCalled()
            // Event should not be produced to output topic
            expect(getProducedEvents()).toHaveLength(0)
            // Event should be sent to DLQ
            const dlqMessages = getDLQMessages()
            expect(dlqMessages).toHaveLength(1)
        })

        it('sends all batch events to DLQ on non-retriable Cymbal error', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Non-retriable error affects the entire batch
            const nonRetriableError = new Error('Bad request - invalid batch')
            ;(nonRetriableError as any).isRetriable = false
            mockCymbalClient.processExceptions.mockRejectedValue(nonRetriableError)

            const messages = [
                createKafkaMessage({ distinctId: 'user-1' }),
                createKafkaMessage({ distinctId: 'user-2' }),
                createKafkaMessage({ distinctId: 'user-3' }),
            ]

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, messages)

            // Non-retriable errors should not be retried
            expect(mockCymbalClient.processExceptions).toHaveBeenCalledTimes(1)
            // All events should be sent to DLQ
            const dlqMessages = getDLQMessages()
            expect(dlqMessages).toHaveLength(3)
            // No events should be produced to output topic
            expect(getProducedEvents()).toHaveLength(0)
        })

        it('runs Hog transformations on events', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Verify the transformer was called
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)
        })

        it('preserves GeoIP properties added by Hog transformations', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            // Mock the transformer to add GeoIP properties (simulating the GeoIP transformation)
            mockHogTransformer.transformEventAndProduceMessages.mockImplementation((event) =>
                Promise.resolve({
                    event: {
                        ...event,
                        properties: {
                            ...event.properties,
                            $geoip_country_code: 'US',
                            $geoip_city_name: 'San Francisco',
                            $geoip_subdivision_1_code: 'CA',
                            $geoip_subdivision_1_name: 'California',
                            $geoip_latitude: 37.7749,
                            $geoip_longitude: -122.4194,
                        },
                    },
                    invocationResults: [],
                })
            )

            const message = createKafkaMessage({ ip: '89.160.20.129' })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)

            const props = parseJSON(producedEvents[0].properties)
            expect(props.$geoip_country_code).toBe('US')
            expect(props.$geoip_city_name).toBe('San Francisco')
            expect(props.$geoip_subdivision_1_code).toBe('CA')
            expect(props.$geoip_subdivision_1_name).toBe('California')
            expect(props.$geoip_latitude).toBe(37.7749)
            expect(props.$geoip_longitude).toBe(-122.4194)
        })

        it('preserves original event properties through enrichment', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Cymbal receives original properties and returns them with fingerprint/issue_id
            const cymbalResponse = createCymbalResponseWithEnrichedProperties({
                $exception_list: [{ type: 'Error', value: 'Test error' }],
                custom_property: 'should-be-preserved',
                $browser: 'Chrome',
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            const message = createKafkaMessage({
                properties: {
                    custom_property: 'should-be-preserved',
                    $browser: 'Chrome',
                },
            })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)
            const props = parseJSON(producedEvents[0].properties)
            expect(props.custom_property).toBe('should-be-preserved')
            expect(props.$browser).toBe('Chrome')
        })

        it('emits ingestion warning when Cymbal returns $cymbal_errors', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            const eventUuid = new UUIDT().toString()

            // Cymbal returns event with processing errors attached
            const cymbalResponse = createCymbalResponse({
                uuid: eventUuid,
                properties: {
                    $exception_list: [{ type: 'Error', value: 'Test error' }],
                    $exception_fingerprint: 'test-fingerprint',
                    $exception_issue_id: 'test-issue-id',
                    $cymbal_errors: [
                        'No sourcemap found for source url: https://example.com/app.js',
                        'Token not found for frame: app.js:10:5',
                    ],
                },
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            const message = createKafkaMessage({ eventUuid })

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Event should still be emitted (errors don't block processing)
            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)

            // Verify ingestion warning was sent
            const warnings = getIngestionWarnings()
            expect(warnings).toHaveLength(1)
            expect(warnings[0].type).toBe('error_tracking_exception_processing_errors')
            expect(warnings[0].team_id).toBe(123)

            const details = parseJSON(warnings[0].details)
            expect(details.eventUuid).toBe(eventUuid)
            expect(details.errors).toEqual([
                'No sourcemap found for source url: https://example.com/app.js',
                'Token not found for frame: app.js:10:5',
            ])
        })

        it('does not emit ingestion warning when Cymbal returns no errors', async () => {
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)

            // Cymbal returns event without errors
            const cymbalResponse = createCymbalResponse({
                properties: {
                    $exception_list: [{ type: 'Error', value: 'Test error' }],
                    $exception_fingerprint: 'test-fingerprint',
                    $exception_issue_id: 'test-issue-id',
                    // No $cymbal_errors
                },
            })
            mockCymbalClient.processExceptions.mockResolvedValue([cymbalResponse])

            const message = createKafkaMessage({})

            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            // Event should be emitted
            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)

            // No ingestion warning should be sent
            const warnings = getIngestionWarnings()
            expect(warnings).toHaveLength(0)
        })

        it('always uses full person_mode to preserve group properties', async () => {
            // Error tracking always uses processPerson=true to preserve $group_* properties,
            // unlike the analytics pipeline which may use propertyless mode.

            // Test with person found - should be 'full' mode
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const pipeline1 = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline1, [createKafkaMessage({})])

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            let producedEvents = getProducedEvents()
            expect(producedEvents[0].person_mode).toBe('full')

            // Reset mocks
            mockKafkaProducer.produce.mockClear()
            mockHogTransformer.transformEventAndProduceMessages.mockClear()

            // Test without person found - still uses 'full' mode because processPerson=true
            mockPersonRepository.fetchPerson.mockResolvedValue(undefined)
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const pipeline2 = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline2, [createKafkaMessage({ distinctId: 'no-person' })])

            // Verify Hog transformations were run for second pipeline too
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            producedEvents = getProducedEvents()
            // Error tracking sets processPerson=true so it's always 'full' mode
            expect(producedEvents[0].person_mode).toBe('full')
        })
    })

    describe('cookieless processing', () => {
        const createMockOverflowRedirectService = (
            keysToRedirect: Set<string> = new Set()
        ): jest.Mocked<OverflowRedirectService> =>
            ({
                handleEventBatch: jest.fn().mockResolvedValue(keysToRedirect),
                healthCheck: jest.fn().mockResolvedValue({ status: 'ok' }),
                shutdown: jest.fn().mockResolvedValue(undefined),
            }) as unknown as jest.Mocked<OverflowRedirectService>

        it('invokes the cookieless manager once per batch', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse(), createCymbalResponse()])

            const messages = [createKafkaMessage({ distinctId: 'a' }), createKafkaMessage({ distinctId: 'b' })]
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, messages)

            expect(mockCookielessManager.doBatch).toHaveBeenCalledTimes(1)
            const passedEvents = mockCookielessManager.doBatch.mock.calls[0][0]
            expect(passedEvents).toHaveLength(2)
            expect(passedEvents.map((e: any) => e.event.distinct_id)).toEqual(['a', 'b'])
        })

        it('uses the rewritten distinct_id from the cookieless manager downstream', async () => {
            const person = createTestPerson({ distinct_id: 'hashed-distinct-id' })
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            mockCookielessManager.doBatch.mockImplementationOnce((events: any[]) =>
                Promise.resolve(
                    events.map((e) =>
                        ok({
                            ...e,
                            event: { ...e.event, distinct_id: 'hashed-distinct-id' },
                        })
                    )
                )
            )

            const message = createKafkaMessage({ distinctId: '$posthog_cookieless' })
            const pipeline = createErrorTrackingPipeline(pipelineConfig)
            await runErrorTrackingPipeline(pipeline, [message])

            const producedEvents = getProducedEvents()
            expect(producedEvents).toHaveLength(1)
            expect(producedEvents[0].distinct_id).toBe('hashed-distinct-id')
        })

        it('redirects cookieless events to overflow when their message key is flagged', async () => {
            // The rate limit step keys on the Kafka message key, so a flagged
            // cookieless event goes to overflow before Cymbal runs.
            const flagging = createMockOverflowRedirectService(new Set([`test-token-123:${COOKIELESS_SENTINEL_VALUE}`]))
            const configWithOverflow: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                overflowMode: 'redirect',
                overflowRedirectService: flagging,
            }

            const message = createKafkaMessage({ distinctId: COOKIELESS_SENTINEL_VALUE })
            const pipeline = createErrorTrackingPipeline(configWithOverflow)
            await runErrorTrackingPipeline(pipeline, [message])

            expect(mockCymbalClient.processExceptions).not.toHaveBeenCalled()
            expect(getProducedEvents()).toHaveLength(0)
            expect(getOverflowMessages()).toHaveLength(1)
        })
    })

    describe('TopHog metrics', () => {
        let mockTophogQueueMessages: jest.Mock
        let topHog: TopHog

        beforeEach(() => {
            mockTophogQueueMessages = jest.fn().mockResolvedValue(undefined)
            const tophogOutputs = {
                queueMessages: mockTophogQueueMessages,
            } as unknown as IngestionOutputs<TophogOutput>

            topHog = new TopHog({
                outputs: tophogOutputs,
                pipeline: 'errortracking',
                lane: 'main',
            })
        })

        const getTopHogMessages = (): any[] => {
            return mockTophogQueueMessages.mock.calls.flatMap((call: any) =>
                call[1].map((m: any) => parseJSON(m.value.toString()))
            )
        }

        it('records messages_by_token metric for parsed messages', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const configWithTopHog: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                topHog,
            }

            const pipeline = createErrorTrackingPipeline(configWithTopHog)
            await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])
            await topHog.flush()

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const messages = getTopHogMessages()
            const messagesByToken = messages.find((m) => m.metric === 'messages_by_token')
            expect(messagesByToken).toBeDefined()
            expect(messagesByToken.type).toBe('sum')
            expect(messagesByToken.key.token).toBe('test-token-123')
            expect(messagesByToken.value).toBe(1)
        })

        it('records emitted_events metric when events are emitted', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const configWithTopHog: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                topHog,
            }

            const pipeline = createErrorTrackingPipeline(configWithTopHog)
            await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])
            await topHog.flush()

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const messages = getTopHogMessages()
            const emittedEventsMetric = messages.find((m) => m.metric === 'emitted_events')
            expect(emittedEventsMetric).toBeDefined()
            expect(emittedEventsMetric.type).toBe('sum')
            expect(emittedEventsMetric.key.team_id).toBe('123')
            expect(emittedEventsMetric.value).toBe(1)
        })

        it('records emitted_events_per_distinct_id metric with distinct_id', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const configWithTopHog: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                topHog,
            }

            const pipeline = createErrorTrackingPipeline(configWithTopHog)
            await runErrorTrackingPipeline(pipeline, [createKafkaMessage({ distinctId: 'specific-user' })])
            await topHog.flush()

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const messages = getTopHogMessages()
            const perDistinctIdMetric = messages.find((m) => m.metric === 'emitted_events_per_distinct_id')
            expect(perDistinctIdMetric).toBeDefined()
            expect(perDistinctIdMetric.type).toBe('sum')
            expect(perDistinctIdMetric.key.team_id).toBe('123')
            expect(perDistinctIdMetric.key.distinct_id).toBe('specific-user')
            expect(perDistinctIdMetric.value).toBe(1)
        })

        it('includes pipeline and lane labels in TopHog output', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockResolvedValue([createCymbalResponse()])

            const configWithTopHog: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                topHog,
            }

            const pipeline = createErrorTrackingPipeline(configWithTopHog)
            await runErrorTrackingPipeline(pipeline, [createKafkaMessage({})])
            await topHog.flush()

            // Verify Hog transformations were run
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(1)

            const messages = getTopHogMessages()
            expect(messages.length).toBeGreaterThan(0)
            for (const msg of messages) {
                expect(msg.pipeline).toBe('errortracking')
                expect(msg.lane).toBe('main')
            }
        })

        it('aggregates metrics across multiple events', async () => {
            const person = createTestPerson()
            mockPersonRepository.fetchPersonsByDistinctIds.mockResolvedValue([person])
            mockCymbalClient.processExceptions.mockImplementation((events) =>
                Promise.resolve(events.map(() => createCymbalResponse()))
            )

            const configWithTopHog: ErrorTrackingPipelineConfig = {
                ...pipelineConfig,
                topHog,
            }

            const messages = [
                createKafkaMessage({ distinctId: 'user-1' }),
                createKafkaMessage({ distinctId: 'user-2' }),
                createKafkaMessage({ distinctId: 'user-1' }),
            ]

            const pipeline = createErrorTrackingPipeline(configWithTopHog)
            await runErrorTrackingPipeline(pipeline, messages)
            await topHog.flush()

            // Verify Hog transformations were run for all events
            expect(mockHogTransformer.transformEventAndProduceMessages).toHaveBeenCalledTimes(3)

            const topHogMessages = getTopHogMessages()

            // messages_by_token should have count=3 (one per parsed message)
            const messagesByToken = topHogMessages.find((m) => m.metric === 'messages_by_token')
            expect(messagesByToken.value).toBe(3)

            // emitted_events should have value=3
            const emittedEventsMetric = topHogMessages.find((m) => m.metric === 'emitted_events')
            expect(emittedEventsMetric.value).toBe(3)

            // emitted_events_per_distinct_id should have two entries (user-1 with 2, user-2 with 1)
            const perDistinctIdMetrics = topHogMessages.filter((m) => m.metric === 'emitted_events_per_distinct_id')
            const user1Metric = perDistinctIdMetrics.find((m) => m.key.distinct_id === 'user-1')
            const user2Metric = perDistinctIdMetrics.find((m) => m.key.distinct_id === 'user-2')
            expect(user1Metric.value).toBe(2)
            expect(user2Metric.value).toBe(1)
        })
    })
})