Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.8 KB
Raw Blame
// Serial: asserts an event-loop lag ceiling, so a worker competing for CPU fails it.
import { mockProducer, mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'

import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'

import { KAFKA_APP_METRICS_2, KAFKA_LOGS_CLICKHOUSE, KAFKA_LOGS_INGESTION_DLQ } from '~/common/config/kafka-topics'
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { UsageRecordBatch } from '~/common/usage-ingestion/usage-record-batch'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'

import { LogRecord, encodeLogRecords } from './log-record-avro'
import { LogsIngestionConsumer } from './logs-ingestion-consumer'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'

jest.setTimeout(30000)

let offsetIncrementer = 0

const createKafkaMessages = async (bodies: string[], headers: Record<string, string>): Promise<Message[]> => {
    const avro = require('avsc')
    const logRecordType = avro.Type.forSchema({
        type: 'record',
        name: 'LogRecord',
        fields: [
            { name: 'uuid', type: ['null', 'string'] },
            { name: 'trace_id', type: ['null', 'bytes'] },
            { name: 'span_id', type: ['null', 'bytes'] },
            { name: 'trace_flags', type: ['null', 'int'] },
            { name: 'timestamp', type: ['null', 'long'] },
            { name: 'observed_timestamp', type: ['null', 'long'] },
            { name: 'body', type: ['null', 'string'] },
            { name: 'severity_text', type: ['null', 'string'] },
            { name: 'severity_number', type: ['null', 'int'] },
            { name: 'service_name', type: ['null', 'string'] },
            { name: 'resource_attributes', type: ['null', { type: 'map', values: 'string' }] },
            { name: 'instrumentation_scope', type: ['null', 'string'] },
            { name: 'event_name', type: ['null', 'string'] },
            { name: 'attributes', type: ['null', { type: 'map', values: 'string' }] },
        ],
    })

    return await Promise.all(
        bodies.map(async (body) => {
            const record: LogRecord = {
                uuid: `test-uuid-${offsetIncrementer}`,
                trace_id: null,
                span_id: null,
                trace_flags: null,
                timestamp: DateTime.now().toMillis() * 1000,
                observed_timestamp: DateTime.now().toMillis() * 1000,
                body,
                severity_text: 'info',
                severity_number: 9,
                service_name: 'test-service',
                resource_attributes: null,
                instrumentation_scope: null,
                event_name: null,
                attributes: null,
            }
            const value = await encodeLogRecords(logRecordType, 'zstandard', [record])
            return {
                key: null,
                value,
                size: value.length,
                topic: 'test',
                offset: offsetIncrementer++,
                timestamp: DateTime.now().toMillis(),
                partition: 1,
                headers: Object.entries(headers).map(([key, headerValue]) => ({
                    [key]: Buffer.from(headerValue),
                })),
            }
        })
    )
}

describe('LogsIngestionConsumer thread relief', () => {
    let consumer: LogsIngestionConsumer
    let hub: Hub
    let team: Team

    beforeEach(async () => {
        offsetIncrementer = 0
        hub = await createHub()
        team = (await createTestTeamFixture(hub.postgres)).team

        // PII scrub and JSON parse make processLogMessageBuffer do real CPU work per message.
        // Without them it short-circuits and never decodes the buffer.
        await hub.postgres.query(
            PostgresUse.COMMON_WRITE,
            `UPDATE posthog_team SET logs_settings = $1 WHERE id = $2`,
            [JSON.stringify({ pii_scrub_logs: true, json_parse_logs: true }), team.id],
            'updateTeamLogsForThreadRelief'
        )

        consumer = new LogsIngestionConsumer(hub, {
            ...hub,
            outputs: new IngestionOutputs<AppMetricsOutput | LogsOutput | LogsDlqOutput>({
                [APP_METRICS_OUTPUT]: new SingleIngestionOutput(
                    APP_METRICS_OUTPUT,
                    KAFKA_APP_METRICS_2,
                    mockProducer,
                    'test'
                ),
                [LOGS_OUTPUT]: new SingleIngestionOutput(LOGS_OUTPUT, KAFKA_LOGS_CLICKHOUSE, mockProducer, 'test'),
                [LOGS_DLQ_OUTPUT]: new SingleIngestionOutput(
                    LOGS_DLQ_OUTPUT,
                    KAFKA_LOGS_INGESTION_DLQ,
                    mockProducer,
                    'test'
                ),
            }),
            usageBatch: new UsageRecordBatch(null, { unit: 'bytes', isTeamEnabled: () => false }),
        })
        consumer['kafkaConsumer'] = {
            connect: jest.fn(),
            disconnect: jest.fn(),
            isHealthy: jest.fn().mockReturnValue({ status: 'healthy' }),
        } as any
        await consumer.start()

        jest.spyOn(hub.quotaLimiting, 'isTeamTokenQuotaLimited').mockResolvedValue(false)
    })

    afterEach(async () => {
        await consumer.stop()
        await closeHub(hub)
    })

    it('should process large batches without blocking the main thread', async () => {
        const body = JSON.stringify({
            user_id: 'usr_abc123',
            email: '[email protected]',
            nested: { a: 1, b: 'two', c: [1, 2, 3, 4, 5] },
            message: 'A long log message ' + 'x'.repeat(500),
        })

        const numberToTest = 2000
        const messages = await createKafkaMessages(
            Array.from({ length: numberToTest }, () => body),
            {
                token: team.api_token,
            }
        )

        let lastCheck = Date.now()
        let longestDelay = 0
        const interval = setInterval(() => {
            longestDelay = Math.max(longestDelay, Date.now() - lastCheck)
            lastCheck = Date.now()
        }, 0)

        try {
            await (
                await consumer.processKafkaBatch(messages)
            ).backgroundTask
        } finally {
            clearInterval(interval)
        }

        // Counted rather than decoded: the logs output carries AVRO, not JSON.
        const produced = mockProducerObserver
            .getProducedMessages()
            .filter((batch) => batch.topic === KAFKA_LOGS_CLICKHOUSE)
            .flatMap((batch) => batch.messages)
        expect(produced).toHaveLength(numberToTest)
        expect(longestDelay).toBeLessThan(120)
    })
})