Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.7 KB
Raw Blame
import { Message } from 'node-rdkafka'

import { sanitizeEvent } from '~/common/utils/event'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { UUID } from '~/common/utils/utils'
import { TopHogMetricFactory, count, max, sum, timer } from '~/ingestion/framework/extensions/tophog'
import { dlq, ok } from '~/ingestion/framework/results'
import { ProcessingStep } from '~/ingestion/framework/steps'
import { EventHeaders, IncomingEvent, PipelineEvent } from '~/types'

/**
 * TopHog metrics for the parse step: parse timing and raw message sizes,
 * keyed by token (the team is not resolved yet at parse time). Sizes are
 * recorded from the raw Kafka message, so malformed events that DLQ at parse
 * still show up — this is the earliest per-sender visibility the pipeline
 * has. Wire via `.parseMessage({ wrap: (step) => topHog(step, parseMessageTopHogMetrics()) })`.
 */
export function parseMessageTopHogMetrics<
    TInput extends { message: Message; headers: EventHeaders },
    TOutput,
>(): TopHogMetricFactory<TInput, TOutput>[] {
    const byToken = (input: TInput) => ({ token: input.headers.token ?? 'unknown' })
    const messageBytes = (input: TInput) => input.message.value?.length ?? 0
    return [
        count('messages_by_token', byToken),
        timer('parse_time_ms_by_token', byToken),
        sum('message_size_by_token', byToken, messageBytes),
        max('max_message_size_by_token', byToken, messageBytes),
        sum(
            'message_size_by_token_per_partition',
            (input) => ({ ...byToken(input), partition: String(input.message.partition) }),
            messageBytes
        ),
    ]
}

export function createParseKafkaMessageStep<T extends { message: Message }>(): ProcessingStep<
    T,
    T & { event: IncomingEvent }
> {
    return function parseKafkaMessageStep(input) {
        const { message } = input

        try {
            const {
                data: dataStr,
                token: _token,
                ...rawEvent
            } = requireObject(parseJSON(message.value!.toString()), 'outer message')
            const data = requireObject(parseJSON(requireString(dataStr, 'data')), 'data field')
            const combinedEvent: Record<string, unknown> = { ...data, ...rawEvent }

            if (!combinedEvent.uuid || !UUID.validateString(combinedEvent.uuid, false)) {
                const cause = combinedEvent.uuid ? 'invalid_uuid' : 'empty_uuid'
                return Promise.resolve(dlq(cause))
            }

            // Use sanitize-only normalization here. Full normalization (including
            // personInitialAndUTMProperties) happens after transformations in normalizeEventStep.
            const event: PipelineEvent = sanitizeEvent(parsePipelineEvent(combinedEvent))

            return Promise.resolve(ok({ ...input, event: { event } }))
        } catch (error) {
            logger.warn('Failed to parse Kafka message', { error })
            return Promise.resolve(
                dlq('failed_parse_message', error instanceof Error ? error : new Error(String(error)))
            )
        }
    }
}

function parsePipelineEvent(raw: Record<string, unknown>): PipelineEvent {
    return {
        uuid: requireString(raw.uuid, 'uuid'),
        distinct_id: requireString(raw.distinct_id, 'distinct_id'),
        event: requireString(raw.event, 'event'),
        ip: optionalString(raw.ip, 'ip', null),
        site_url: optionalString(raw.site_url, 'site_url', ''),
        now: optionalString(raw.now, 'now', ''),
        properties: optionalObject(raw.properties, 'properties') ?? {},
        team_id: optionalNumber(raw.team_id, 'team_id'),
        sent_at: optionalString(raw.sent_at, 'sent_at'),
        timestamp: optionalString(raw.timestamp, 'timestamp'),
        offset: optionalNumber(raw.offset, 'offset'),
        $set: optionalObject(raw.$set, '$set'),
        $set_once: optionalObject(raw.$set_once, '$set_once'),
    }
}

function requireString(value: unknown, field: string): string {
    if (typeof value === 'string') {
        return value
    }
    if (typeof value === 'number' || typeof value === 'boolean') {
        return String(value)
    }
    throw new Error(`expected string for ${field}, got ${typeof value}`)
}

function optionalString(value: unknown, field: string): string | undefined
function optionalString(value: unknown, field: string, fallback: string): string
function optionalString(value: unknown, field: string, fallback: null): string | null
function optionalString(value: unknown, field: string, fallback?: string | null): string | null | undefined {
    if (value == null) {
        return fallback
    }
    return requireString(value, field)
}

function requireNumber(value: unknown, field: string): number {
    if (typeof value === 'string') {
        const parsed = Number(value)
        if (!Number.isFinite(parsed)) {
            throw new Error(`expected number for ${field}, got non-numeric string`)
        }
        return parsed
    }
    if (typeof value !== 'number') {
        throw new Error(`expected number for ${field}, got ${typeof value}`)
    }
    return value
}

function optionalNumber(value: unknown, field: string): number | undefined {
    return value == null ? undefined : requireNumber(value, field)
}

function requireObject(value: unknown, field: string): Record<string, unknown> {
    if (typeof value !== 'object' || value === null || Array.isArray(value)) {
        throw new Error(`expected object for ${field}, got ${Array.isArray(value) ? 'array' : typeof value}`)
    }
    return value as Record<string, unknown>
}

function optionalObject(value: unknown, field: string): Record<string, any> | undefined {
    return value == null ? undefined : requireObject(value, field)
}