Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

19.4 KB
Raw Blame
import { createHash } from 'crypto'
import { Message } from 'node-rdkafka'

import { MessageKey } from '~/common/kafka/producer'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { IngestionOutputMessage } from '~/common/outputs/types'
import { DependencyUnavailableError } from '~/common/utils/db/error'
import { parseJSON } from '~/common/utils/json-parse'

import { LogsIngestionFrameworkConsumer } from '../framework/logs-ingestion-framework-consumer'
import { TracesIngestionFrameworkConsumer } from '../framework/traces-ingestion-framework-consumer'
import { MESSAGE_KEYS } from '../log-pattern-mask'
import { decodeLogRecords } from '../log-record-avro'
import { LogsIngestionConsumer, LogsIngestionConsumerDeps } from '../logs-ingestion-consumer'
import { type MetricRuleRow, compileMetricRules } from '../metrics-rules/compile-metric-rules'
import { type RetentionRuleRow, compileRetentionRuleSet } from '../retention/compile-retention-rules'
import { type SamplingRuleRow, compileRuleSet } from '../sampling/compile-rules'
import { TracesIngestionConsumer, TracesIngestionConsumerDeps } from '../traces-ingestion-consumer'
import type {
    ParityError,
    ParityExpected,
    ParityMessage,
    ParityOutputMessage,
    ParityScenario,
    ParityValue,
} from './parity-types'

const LibrdKafkaError = require('node-rdkafka/lib/error')
const { err2str } = require('node-rdkafka/librdkafka') as { err2str: (code: number) => string }

export function librdKafkaError(code: number, sync = false): Error {
    return LibrdKafkaError.create(sync ? code : new Error(err2str(code)))
}

function toError(spec: ParityError): Error {
    if (spec.kafka_code !== undefined) {
        return librdKafkaError(spec.kafka_code, spec.kafka_sync)
    }
    const cause = new Error(spec.message)
    cause.name = spec.name
    if (spec.dependency_unavailable) {
        return new DependencyUnavailableError(spec.message, 'parity', cause)
    }
    return cause
}

type CapturedMessage = { batch: number; output: string; message: IngestionOutputMessage }

class CapturingIngestionOutputs extends IngestionOutputs<string> {
    public readonly captured: CapturedMessage[] = []
    public producedCount = 0
    public currentBatch = 0
    private readonly remainingFaults: number[]

    constructor(
        private readonly scenario: ParityScenario,
        private readonly discard = false
    ) {
        super({})
        this.remainingFaults = scenario.produce_faults.map((fault) => fault.times)
    }

    private faultFor(output: string, message: IngestionOutputMessage): Error | null {
        for (let i = 0; i < this.scenario.produce_faults.length; i++) {
            const fault = this.scenario.produce_faults[i]
            if (fault.output !== output || this.remainingFaults[i] <= 0) {
                continue
            }
            if (fault.team_id !== undefined && message.headers?.team_id !== fault.team_id) {
                continue
            }
            this.remainingFaults[i]--
            return toError(fault.error)
        }
        return null
    }

    override async produce(output: string, message: IngestionOutputMessage & { key: MessageKey }): Promise<void> {
        await this.queueMessages(output, [message])
    }

    override queueMessages(output: string, messages: IngestionOutputMessage[]): Promise<void> {
        for (const message of messages) {
            const fault = this.faultFor(output, message)
            if (fault) {
                return Promise.reject(fault)
            }
        }
        this.producedCount += messages.length
        if (!this.discard) {
            for (const message of messages) {
                this.captured.push({ batch: this.currentBatch, output, message })
            }
        }
        return Promise.resolve()
    }
}

export function toKafkaMessage(message: ParityMessage): Message {
    const value = message.value_b64 === null ? null : Buffer.from(message.value_b64, 'base64')
    return {
        topic: message.topic,
        partition: message.partition,
        offset: message.offset,
        timestamp: message.timestamp,
        key: message.key === null ? null : Buffer.from(message.key),
        size: value?.length ?? 0,
        value,
        headers: message.headers.map(([key, headerValue]) => ({ [key]: Buffer.from(headerValue) })),
    }
}

/** Avro maps carry no order that ClickHouse reads, so keys are sorted. Avro arrays are ordered, so arrays keep their order. */
export function canonicalizeAvroValue(value: unknown): unknown {
    if (Buffer.isBuffer(value)) {
        return { $bytes: value.toString('hex') }
    }
    if (Array.isArray(value)) {
        return value.map(canonicalizeAvroValue)
    }
    if (value !== null && typeof value === 'object') {
        const out: Record<string, unknown> = {}
        for (const key of Object.keys(value).sort()) {
            out[key] = canonicalizeAvroValue((value as Record<string, unknown>)[key])
        }
        return out
    }
    return value
}

function readContainerSchemaAndCodec(buffer: Buffer): { schema: unknown; codec: string } | null {
    // Read the container header directly to keep the schema JSON exactly as the writer emitted it.
    if (buffer.length < 4 || buffer.subarray(0, 4).toString('latin1') !== 'Obj\u0001') {
        return null
    }
    let offset = 4
    const readLong = (): number => {
        let n = 0
        let shift = 0
        let byte: number
        do {
            byte = buffer[offset++]
            n += (byte & 0x7f) * 2 ** shift
            shift += 7
        } while (byte & 0x80)
        return n % 2 === 0 ? n / 2 : -(n + 1) / 2
    }
    const meta: Record<string, string> = {}
    for (;;) {
        let count = readLong()
        if (count === 0) {
            break
        }
        if (count < 0) {
            count = -count
            readLong()
        }
        for (let i = 0; i < count; i++) {
            const keyLength = readLong()
            const key = buffer.subarray(offset, offset + keyLength).toString('utf8')
            offset += keyLength
            const valueLength = readLong()
            meta[key] = buffer.subarray(offset, offset + valueLength).toString('utf8')
            offset += valueLength
        }
    }
    return { schema: parseJSON(meta['avro.schema']), codec: meta['avro.codec'] ?? 'null' }
}

async function canonicalizeValue(
    output: string,
    value: Buffer | null,
    inputs: Map<string, { batch: number; index: number }>
): Promise<ParityValue> {
    if (value === null) {
        return { kind: 'null' }
    }
    // By bytes, not identity: a copy of the input is still a passthrough.
    const input = inputs.get(value.toString('base64'))
    if (input) {
        return { kind: 'passthrough', ...input }
    }
    if (output === 'logs') {
        const header = readContainerSchemaAndCodec(value)
        if (header) {
            const [, , records] = await decodeLogRecords(value)
            return {
                kind: 'avro',
                schema: header.schema,
                codec: header.codec,
                records: records.map(canonicalizeAvroValue),
            }
        }
    }
    if (output === 'app_metrics') {
        const raw = value.toString('utf8')
        return { kind: 'json', raw, json: canonicalizeAvroValue(parseJSON(raw)) }
    }
    return { kind: 'bytes', sha256: createHash('sha256').update(value).digest('hex'), length: value.length }
}

function sortedHeaders(headers: Record<string, string> | undefined): Record<string, string> {
    const out: Record<string, string> = {}
    for (const key of Object.keys(headers ?? {}).sort()) {
        out[key] = headers![key]
    }
    return out
}

function compareCanonical(a: unknown, b: unknown): number {
    const left = JSON.stringify(a)
    const right = JSON.stringify(b)
    return left < right ? -1 : left > right ? 1 : 0
}

/** These follow concurrent message completion, so they are sorted. A histogram's bucket arrays are positional and keep their order. */
// Database 9, so a parity or benchmark run never flushes the keys a local stack keeps in database 0.
// nosemgrep: trailofbits.generic.redis-unencrypted-transport.redis-unencrypted-transport
export const PARITY_REDIS_URL = 'redis://127.0.0.1:6379/9'

const UNORDERED_METRIC_ARRAYS = new Set([
    'resourceMetrics',
    'scopeMetrics',
    'metrics',
    'dataPoints',
    'attributes',
    'exemplars',
    'filteredAttributes',
])

function canonicalizeMetricPayload(payload: unknown): unknown {
    const sortArrays = (value: unknown, key?: string): unknown => {
        if (Array.isArray(value)) {
            const items = value.map((item) => sortArrays(item))
            return key !== undefined && UNORDERED_METRIC_ARRAYS.has(key) ? items.sort(compareCanonical) : items
        }
        if (value !== null && typeof value === 'object') {
            return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, sortArrays(v, k)]))
        }
        return value
    }
    return sortArrays(canonicalizeAvroValue(payload))
}

function indexInputs(scenario: ParityScenario): Map<string, { batch: number; index: number }> {
    const inputs = new Map<string, { batch: number; index: number }>()
    scenario.batches.forEach((batch, batchIndex) =>
        batch.forEach((message, index) => {
            if (message.value_b64 === null) {
                return
            }
            if (inputs.has(message.value_b64)) {
                throw new Error(
                    `scenario ${scenario.name} repeats an input value at batch ${batchIndex} index ${index}`
                )
            }
            inputs.set(message.value_b64, { batch: batchIndex, index })
        })
    )
    return inputs
}

export type ParityConsumerClasses = {
    logs: new (config: any, deps: any) => Pick<LogsIngestionConsumer, 'processKafkaBatch'>
    traces: new (config: any, deps: any) => Pick<LogsIngestionConsumer, 'processKafkaBatch'>
}

export const CURRENT_CLASSES: ParityConsumerClasses = { logs: LogsIngestionConsumer, traces: TracesIngestionConsumer }

export const FRAMEWORK_CLASSES: ParityConsumerClasses = {
    logs: LogsIngestionFrameworkConsumer,
    traces: TracesIngestionFrameworkConsumer,
}

export function buildParityConsumer(
    scenario: ParityScenario,
    options: { classes?: ParityConsumerClasses; discard?: boolean } = {}
) {
    const classes = options.classes ?? CURRENT_CLASSES
    const outputs = new CapturingIngestionOutputs(scenario, options.discard)
    const teamsById = new Map(scenario.teams.map((team) => [team.id, team]))
    const teamsByToken = new Map(scenario.teams.map((team) => [team.api_token, team]))
    const teamsIngested: number[][] = []
    const metricEmits: ParityExpected['metric_emits'] = []
    const usageRecords: (Omit<ParityExpected['usage_records'][number], 'record_group'> & { record_id: string })[] = []
    const quotaChecks = new Set<string>()
    const teamByIdFaultsLeft = new Map(
        Object.entries(scenario.team_by_id_faults).map(([teamId, fault]) => [Number(teamId), fault.times])
    )
    const patternFaultsLeft = new Map(
        Object.entries(scenario.pattern_keys_faults).map(([teamId, fault]) => [Number(teamId), fault.times])
    )

    // Rules compile once per team, as the production caches serve them, so a benchmark does not time compilation.
    const compiled = new Map<string, unknown>()
    const memo = <T>(key: string, build: () => T): T => {
        if (!compiled.has(key)) {
            compiled.set(key, build())
        }
        return compiled.get(key) as T
    }

    const teamManager = {
        getTeam: (id: number) => {
            const left = teamByIdFaultsLeft.get(id) ?? 0
            if (left > 0) {
                teamByIdFaultsLeft.set(id, left - 1)
                return Promise.reject(toError(scenario.team_by_id_faults[id]))
            }
            if (scenario.team_by_id_missing.includes(id)) {
                return Promise.resolve(null)
            }
            return Promise.resolve(teamsById.get(id) ?? null)
        },
        getTeamByToken: (token: string) => {
            const fault = scenario.team_lookup_faults[token]
            if (fault) {
                return Promise.reject(toError(fault))
            }
            return Promise.resolve(teamsByToken.get(token) ?? null)
        },
        setTeamsIngestedData: (ids: number[]) => {
            if (!options.discard) {
                teamsIngested.push([...ids])
            }
            return Promise.resolve()
        },
    }

    const deps: TracesIngestionConsumerDeps = {
        teamManager: teamManager as unknown as LogsIngestionConsumerDeps['teamManager'],
        quotaLimiting: {
            isTeamTokenQuotaLimited: (token: string, resource: string) => {
                if (!options.discard) {
                    quotaChecks.add(JSON.stringify([token, resource]))
                }
                return Promise.resolve(scenario.quota_limited_tokens.includes(token))
            },
        } as unknown as LogsIngestionConsumerDeps['quotaLimiting'],
        outputs: outputs as unknown as LogsIngestionConsumerDeps['outputs'],
        usageBatch: {
            add: (teamId: number, metric: string, recordId: string, quantity = 1, unit?: string) => {
                if (options.discard) {
                    return
                }
                usageRecords.push({
                    batch: outputs.currentBatch,
                    team_id: teamId,
                    metric,
                    quantity,
                    unit: unit ?? '',
                    record_id: recordId,
                })
            },
            flush: () => Promise.resolve(),
        } as unknown as LogsIngestionConsumerDeps['usageBatch'],
        dependencyRetry: { retryCount: 2, initialRetryDelayMs: 1 },
        samplingRulesCache: {
            getCompiledRuleSet: (teamId: number) =>
                Promise.resolve(
                    memo(`sampling:${teamId}`, () =>
                        compileRuleSet((scenario.sampling_rules[teamId] ?? []) as SamplingRuleRow[])
                    )
                ),
        } as unknown as LogsIngestionConsumerDeps['samplingRulesCache'],
        retentionRulesCache: {
            getCompiledRuleSet: (teamId: number, source: string) =>
                Promise.resolve(
                    memo(`retention:${teamId}:${source}`, () =>
                        compileRetentionRuleSet(
                            (scenario.retention_rules[teamId]?.[source] ?? []) as RetentionRuleRow[]
                        )
                    )
                ),
        } as unknown as LogsIngestionConsumerDeps['retentionRulesCache'],
        metricRulesCache: {
            getCompiledRules: (teamId: number) => {
                const fault = scenario.metric_rules_faults[teamId]
                if (fault) {
                    return Promise.reject(toError(fault))
                }
                return Promise.resolve(
                    memo(`metrics:${teamId}`, () =>
                        compileMetricRules((scenario.metric_rules[teamId] ?? []) as MetricRuleRow[])
                    )
                )
            },
        } as unknown as LogsIngestionConsumerDeps['metricRulesCache'],
        metricsEmitter: {
            emit: (token: string, teamId: number, payload: Record<string, unknown>) => {
                if (scenario.metric_emit_faults.includes(teamId)) {
                    return Promise.reject(new Error('metric emit failed'))
                }
                if (!options.discard) {
                    metricEmits.push({ token, team_id: teamId, payload })
                }
                return Promise.resolve(true)
            },
        } as unknown as LogsIngestionConsumerDeps['metricsEmitter'],
        patternMessageKeysCache: {
            getMessageKeys: (teamId: number) => {
                const left = patternFaultsLeft.get(teamId) ?? 0
                if (left > 0) {
                    patternFaultsLeft.set(teamId, left - 1)
                    return Promise.reject(toError(scenario.pattern_keys_faults[teamId]))
                }
                // Production returns the default keys for a team without a row.
                return Promise.resolve(scenario.pattern_message_keys[teamId] ?? MESSAGE_KEYS)
            },
        } as unknown as LogsIngestionConsumerDeps['patternMessageKeysCache'],
        internalSourcePolicies:
            scenario.internal_source_policies &&
            Object.fromEntries(
                Object.entries(scenario.internal_source_policies).map(([source, policy]) => [
                    source,
                    { ...policy, serviceName: `posthog-internal-${source}` },
                ])
            ),
        tracingConfigCache: {
            getRetentionDays: (teamId: number) => Promise.resolve(scenario.traces_retention_days[teamId] ?? 30),
        } as unknown as TracesIngestionConsumerDeps['tracingConfigCache'],
    }
    for (const dep of scenario.omit_deps) {
        delete deps[dep]
    }

    const config = { ...scenario.config, LOGS_REDIS_HOST: PARITY_REDIS_URL, TRACES_REDIS_HOST: PARITY_REDIS_URL } as any
    const consumer = scenario.consumer === 'traces' ? new classes.traces(config, deps) : new classes.logs(config, deps)
    return { consumer, outputs, teamsIngested, metricEmits, usageRecords, quotaChecks }
}

/** The caller empties `PARITY_REDIS_URL` before the call. */
export async function runParityScenario(
    scenario: ParityScenario,
    setClock: (nowMs: number) => void,
    classes: ParityConsumerClasses = CURRENT_CLASSES
): Promise<ParityExpected> {
    const { consumer, outputs, teamsIngested, metricEmits, usageRecords, quotaChecks } = buildParityConsumer(scenario, {
        classes,
    })
    const inputs = indexInputs(scenario)
    const batchResults: ParityExpected['batches'] = []
    for (let batchIndex = 0; batchIndex < scenario.batches.length; batchIndex++) {
        outputs.currentBatch = batchIndex
        setClock(scenario.batch_now_ms[batchIndex])
        const messages = scenario.batches[batchIndex].map(toKafkaMessage)
        try {
            const { backgroundTask } = await consumer.processKafkaBatch(messages)
            await backgroundTask
            batchResults.push({ error: null })
        } catch (error) {
            batchResults.push({ error: error instanceof Error ? error.name : String(error) })
        }
    }

    const canonicalOutputs: ParityOutputMessage[] = []
    for (const { batch, output, message } of outputs.captured) {
        canonicalOutputs.push({
            batch,
            output,
            key: message.key === null || message.key === undefined ? null : String(message.key),
            headers: sortedHeaders(message.headers),
            value: await canonicalizeValue(output, message.value, inputs),
        })
    }
    canonicalOutputs.sort(compareCanonical)

    usageRecords.sort((a, b) => compareCanonical({ ...a, record_id: '' }, { ...b, record_id: '' }))
    const groups = new Map<string, number>()
    const canonicalUsage = usageRecords.map(({ record_id, ...rest }) => {
        if (!groups.has(record_id)) {
            groups.set(record_id, groups.size)
        }
        return { ...rest, record_group: groups.get(record_id)! }
    })

    const canonicalEmits = metricEmits.map((emit) => ({ ...emit, payload: canonicalizeMetricPayload(emit.payload) }))
    canonicalEmits.sort(compareCanonical)

    return {
        batches: batchResults,
        outputs: canonicalOutputs,
        metric_emits: canonicalEmits,
        usage_records: canonicalUsage,
        teams_ingested: teamsIngested,
        quota_checks: [...quotaChecks].sort().map((pair) => parseJSON(pair) as [string, string]),
    }
}