Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

24.3 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { logger } from '~/common/utils/logger'
import { TopHogRegistry, createTopHogWrapper } from '~/ingestion/framework/extensions/tophog'
import { PipelineResultType, dlq, ok } from '~/ingestion/framework/results'
import { createRecordingTopHog } from '~/tests/helpers/tophog'
import { EventHeaders } from '~/types'

import { createParseKafkaMessageStep, parseMessageTopHogMetrics } from './parse-kafka-message'

// Mock dependencies
jest.mock('~/common/utils/logger')

const VALID_UUID = '123e4567-e89b-12d3-a456-426614174000'

describe('createParseKafkaMessageStep', () => {
    const mockLogger = logger as jest.Mocked<typeof logger>
    let step: ReturnType<typeof createParseKafkaMessageStep>

    beforeEach(() => {
        jest.clearAllMocks()
        step = createParseKafkaMessageStep()
    })

    describe('successful parsing', () => {
        it('should parse valid Kafka message with complete event data', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test_user',
                            team_id: 1,
                            properties: { test: 'value' },
                        }),
                        token: 'test_token',
                        ip: '127.0.0.1',
                        site_url: 'https://example.com',
                        uuid: VALID_UUID,
                        now: '2023-01-01T00:00:00Z',
                        sent_at: '2023-01-01T00:00:01Z',
                        kafka_offset: '123',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(
                ok({
                    message: mockMessage,
                    event: {
                        event: {
                            event: 'test_event',
                            distinct_id: 'test_user',
                            team_id: 1,
                            properties: {
                                test: 'value',
                                $ip: '127.0.0.1',
                                $sent_at: '2023-01-01T00:00:01Z',
                            },
                            ip: null,
                            site_url: 'https://example.com',
                            uuid: VALID_UUID,
                            now: '2023-01-01T00:00:00Z',
                            sent_at: '2023-01-01T00:00:01Z',
                        },
                    },
                })
            )
        })

        it('should parse message with minimal event data', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'minimal_event',
                            distinct_id: 'user123',
                        }),
                        uuid: VALID_UUID,
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(
                ok({
                    message: mockMessage,
                    event: {
                        event: {
                            event: 'minimal_event',
                            distinct_id: 'user123',
                            uuid: VALID_UUID,
                            ip: null,
                            site_url: '',
                            now: '',
                            properties: {},
                        },
                    },
                })
            )
        })

        it('should handle message with additional raw event fields', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test_user',
                        }),
                        uuid: VALID_UUID,
                        custom_field: 'custom_value',
                        another_field: 42,
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(
                ok({
                    message: mockMessage,
                    event: {
                        event: {
                            event: 'test_event',
                            distinct_id: 'test_user',
                            uuid: VALID_UUID,
                            ip: null,
                            site_url: '',
                            now: '',
                            properties: {},
                        },
                    },
                })
            )
        })

        it('should preserve all properties from both data and raw event', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test_user',
                            properties: { inner: 'value' },
                        }),
                        properties: { outer: 'value' },
                        token: 'test_token',
                        uuid: VALID_UUID,
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(
                ok({
                    message: mockMessage,
                    event: {
                        event: {
                            event: 'test_event',
                            distinct_id: 'test_user',
                            uuid: VALID_UUID,
                            properties: { outer: 'value' }, // This will overwrite the inner properties
                            ip: null,
                            site_url: '',
                            now: '',
                        },
                    },
                })
            )
        })

        it('should normalize event data during parsing', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test\u0000user', // with null byte
                            team_id: 1,
                            properties: { test: 'value' },
                            ip: '127.0.0.1',
                        }),
                        token: 'test\u0000token', // with null byte
                        uuid: VALID_UUID,
                        ip: '192.168.1.1',
                        sent_at: '2023-01-01T00:00:01Z',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(
                ok({
                    message: mockMessage,
                    event: {
                        event: {
                            event: 'test_event',
                            distinct_id: 'test\uFFFDuser', // null byte replaced with replacement character
                            team_id: 1,
                            properties: {
                                test: 'value',
                                $ip: '192.168.1.1', // ip from raw event field added to properties
                                $sent_at: '2023-01-01T00:00:01Z', // sent_at added to properties
                            },
                            ip: null, // ip field set to null for safety
                            uuid: VALID_UUID,
                            site_url: '',
                            now: '',
                            sent_at: '2023-01-01T00:00:01Z',
                        },
                    },
                })
            )
        })
    })

    describe('uuid validation', () => {
        it.each([
            ['missing uuid', {}, 'empty_uuid'],
            ['null uuid', { uuid: null }, 'empty_uuid'],
            ['empty string uuid', { uuid: '' }, 'empty_uuid'],
            ['invalid format', { uuid: 'not-a-uuid' }, 'invalid_uuid'],
            ['too short', { uuid: '123e4567-e89b-12d3-a456' }, 'invalid_uuid'],
        ])('should DLQ message with %s', async (_label, uuidOverride, expectedReason) => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test_user',
                        }),
                        ...uuidOverride,
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq(expectedReason))
        })
    })

    describe('error handling', () => {
        it('should return DLQ when message value is null', async () => {
            const mockMessage: Message = {
                value: null,
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })

        it('should return DLQ when message value is undefined', async () => {
            const mockMessage: Message = {
                value: undefined,
            } as unknown as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })

        it('should return DLQ when outer JSON is invalid', async () => {
            const mockMessage: Message = {
                value: Buffer.from('invalid json'),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })

        it('should return DLQ when data field JSON is invalid', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: 'invalid json in data field',
                        token: 'test_token',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })
    })

    describe('type validation', () => {
        function makeMessage(
            dataOverrides: Record<string, unknown> = {},
            outerOverrides: Record<string, unknown> = {}
        ): Message {
            return {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: 'test_user',
                            ...dataOverrides,
                        }),
                        uuid: VALID_UUID,
                        ...outerOverrides,
                    })
                ),
            } as Message
        }

        describe('string fields accept strings, numbers, and booleans', () => {
            it.each([
                ['distinct_id', { distinct_id: 123 }, '123'],
                ['distinct_id', { distinct_id: true }, 'true'],
                ['event', { event: 42 }, '42'],
            ] as const)('%s coerces to string', async (field, dataOverrides, expected) => {
                const result = await step({ message: makeMessage(dataOverrides) })

                expect(result.type).toBe(PipelineResultType.OK)
                const event = (result as any).value.event.event
                expect(event[field]).toBe(expected)
            })

            it('ip with number coerces to string', async () => {
                const result = await step({ message: makeMessage({}, { ip: 10 }) })

                expect(result.type).toBe(PipelineResultType.OK)
                const event = (result as any).value.event.event
                // sanitizeEvent moves ip to properties.$ip and sets ip to null
                expect(event.properties.$ip).toBe('10')
                expect(event.ip).toBeNull()
            })
        })

        describe('string fields reject objects and arrays', () => {
            it.each([
                ['distinct_id', { distinct_id: { a: 1 } }],
                ['distinct_id', { distinct_id: [1, 2] }],
                ['event', { event: { a: 1 } }],
                ['event', { event: ['a'] }],
            ])('%s with %j should DLQ', async (_field, dataOverrides) => {
                const result = await step({ message: makeMessage(dataOverrides) })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })
        })

        describe('number fields accept numbers and numeric strings', () => {
            it.each([
                ['team_id', { team_id: '42' }, 42],
                ['team_id', { team_id: 7 }, 7],
                ['offset', { offset: '100' }, 100],
                ['offset', { offset: 0 }, 0],
            ] as const)('%s coerces to number', async (field, dataOverrides, expected) => {
                const result = await step({ message: makeMessage(dataOverrides) })

                expect(result.type).toBe(PipelineResultType.OK)
                const event = (result as any).value.event.event
                expect(event[field]).toBe(expected)
            })
        })

        describe('number fields reject non-numeric values', () => {
            it.each([
                ['team_id', { team_id: 'abc' }],
                ['team_id', { team_id: true }],
                ['team_id', { team_id: { a: 1 } }],
                ['offset', { offset: 'not-a-number' }],
                ['offset', { offset: [1] }],
            ])('%s with %j should DLQ', async (_field, dataOverrides) => {
                const result = await step({ message: makeMessage(dataOverrides) })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })
        })

        describe('object fields reject non-objects', () => {
            it.each([
                ['properties', { properties: 'string' }],
                ['properties', { properties: 123 }],
                ['properties', { properties: [1, 2] }],
                ['$set', { $set: 'string' }],
                ['$set', { $set: 42 }],
                ['$set_once', { $set_once: true }],
                ['$set_once', { $set_once: [1] }],
            ])('%s with %j should DLQ', async (_field, dataOverrides) => {
                const result = await step({ message: makeMessage(dataOverrides) })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })
        })

        describe('optional fields default correctly when absent', () => {
            it('should use defaults for all optional fields', async () => {
                const result = await step({ message: makeMessage() })

                expect(result.type).toBe(PipelineResultType.OK)
                const event = (result as any).value.event.event
                expect(event.ip).toBeNull()
                expect(event.site_url).toBe('')
                expect(event.now).toBe('')
                expect(event.team_id).toBeUndefined()
                expect(event.sent_at).toBeUndefined()
                expect(event.timestamp).toBeUndefined()
                expect(event.offset).toBeUndefined()
                expect(event.$set).toBeUndefined()
                expect(event.$set_once).toBeUndefined()
            })
        })

        describe('structural validation', () => {
            it.each([
                ['outer message is an array', JSON.stringify([1, 2, 3])],
                ['outer message is a string', JSON.stringify('hello')],
                ['outer message is a number', JSON.stringify(42)],
            ])('should DLQ when %s', async (_label, rawJson) => {
                const msg = { value: Buffer.from(rawJson) } as Message
                const result = await step({ message: msg })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })

            it.each([
                ['data field is an array', JSON.stringify([1, 2])],
                ['data field is a number', JSON.stringify(99)],
                ['data field is null', JSON.stringify(null)],
            ])('should DLQ when %s', async (_label, dataJson) => {
                const msg = {
                    value: Buffer.from(JSON.stringify({ data: dataJson, uuid: VALID_UUID })),
                } as Message
                const result = await step({ message: msg })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })

            it('should DLQ when data field is not a string', async () => {
                const msg = {
                    value: Buffer.from(JSON.stringify({ data: 123, uuid: VALID_UUID })),
                } as Message
                const result = await step({ message: msg })
                expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            })
        })
    })

    describe('parseMessageTopHogMetrics', () => {
        function createWrappedStep(registry: TopHogRegistry) {
            return createTopHogWrapper(registry)(
                createParseKafkaMessageStep<{ message: Message; headers: EventHeaders }>(),
                parseMessageTopHogMetrics()
            )
        }

        const baseHeaders = {
            force_disable_person_processing: false,
            historical_migration: false,
            skip_heatmap_processing: false,
        }

        it('records sizes and timing keyed by token, including for messages that fail to parse', async () => {
            const { registry, records } = createRecordingTopHog()
            const wrapped = createWrappedStep(registry)

            const malformed = Buffer.from('not json')
            const result = await wrapped({
                message: { value: malformed, partition: 7 } as Message,
                headers: { ...baseHeaders, token: 'token-a' },
            })

            // The parse itself DLQ'd — the metrics below were still recorded
            // from the raw message.
            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(records.get('messages_by_token')).toEqual([{ key: { token: 'token-a' }, value: 1 }])
            expect(records.get('message_size_by_token')).toEqual([
                { key: { token: 'token-a' }, value: malformed.length },
            ])
            expect(records.get('max_message_size_by_token')).toEqual([
                { key: { token: 'token-a' }, value: malformed.length },
            ])
            expect(records.get('message_size_by_token_per_partition')).toEqual([
                { key: { token: 'token-a', partition: '7' }, value: malformed.length },
            ])
            const timings = records.get('parse_time_ms_by_token')!
            expect(timings).toHaveLength(1)
            expect(timings[0].key).toEqual({ token: 'token-a' })
            expect(timings[0].value).toBeGreaterThanOrEqual(0)
            expect([...records.keys()].sort()).toEqual([
                'max_message_size_by_token',
                'message_size_by_token',
                'message_size_by_token_per_partition',
                'messages_by_token',
                'parse_time_ms_by_token',
            ])
        })

        it('records a successful parse the same way, falling back to an unknown token key', async () => {
            const { registry, records } = createRecordingTopHog()
            const wrapped = createWrappedStep(registry)

            const value = Buffer.from(
                JSON.stringify({
                    data: JSON.stringify({ event: 'test_event', distinct_id: 'user' }),
                    uuid: VALID_UUID,
                })
            )
            const result = await wrapped({
                message: { value, partition: 0 } as Message,
                headers: baseHeaders,
            })

            expect(result.type).toBe(PipelineResultType.OK)
            expect(records.get('messages_by_token')).toEqual([{ key: { token: 'unknown' }, value: 1 }])
            expect(records.get('message_size_by_token')).toEqual([{ key: { token: 'unknown' }, value: value.length }])
            expect(records.get('max_message_size_by_token')).toEqual([
                { key: { token: 'unknown' }, value: value.length },
            ])
            expect(records.get('message_size_by_token_per_partition')).toEqual([
                { key: { token: 'unknown', partition: '0' }, value: value.length },
            ])
            expect(records.get('parse_time_ms_by_token')![0].key).toEqual({ token: 'unknown' })
        })
    })

    describe('edge cases', () => {
        it('should handle empty data field', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: '',
                        token: 'test_token',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })

        it('should handle data field with only whitespace', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: '   ',
                        token: 'test_token',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
            expect(mockLogger.warn).toHaveBeenCalledWith('Failed to parse Kafka message', {
                error: expect.any(Error),
            })
        })

        it('should DLQ message with empty object in data and no uuid', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({}),
                        token: 'test_token',
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('empty_uuid'))
        })

        it('should DLQ message with null distinct_id', async () => {
            const mockMessage: Message = {
                value: Buffer.from(
                    JSON.stringify({
                        data: JSON.stringify({
                            event: 'test_event',
                            distinct_id: null,
                        }),
                        uuid: VALID_UUID,
                    })
                ),
            } as Message

            const input = { message: mockMessage }
            const result = await step(input)

            expect(result).toEqual(dlq('failed_parse_message', expect.any(Error)))
        })
    })
})