Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.5 KB
Raw Blame
/**
 * NOTE: We are often experimenting with different job queue implementations.
 * To make this easier this class is designed to abstract the queue as much as possible from
 * the underlying implementation.
 */
import { Message } from 'node-rdkafka'
import { compress, uncompress } from 'snappy'

import { KafkaConsumerInterface, createKafkaConsumer } from '~/common/kafka/consumer'
import { KafkaProducerWrapper } from '~/common/kafka/producer'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'

import { HealthCheckResult, HealthCheckResultError } from '../../../types'
import { CdpConfig } from '../../config'
import { CyclotronJobInvocation, CyclotronJobInvocationResult, CyclotronJobQueueKind } from '../../types'
import { JobQueue } from './job-queue.interface'
import { cdpJobSizeCompressedKb, cdpJobSizeKb, createInvocationSanitizer, observeConsumedBatch } from './shared'

export class CyclotronJobQueueKafka implements JobQueue {
    private kafkaConsumer?: KafkaConsumerInterface
    private kafkaProducer?: KafkaProducerWrapper
    private queue?: CyclotronJobQueueKind
    private consumeBatch?: (invocations: CyclotronJobInvocation[]) => Promise<{ backgroundTask: Promise<any> }>
    private sanitizer: ReturnType<typeof createInvocationSanitizer>

    constructor(
        private kafkaClientRack: string | undefined,
        private config: Pick<
            CdpConfig,
            'CDP_CYCLOTRON_COMPRESS_KAFKA_DATA' | 'CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS'
        >,
        private consumerBatchSize: number
    ) {
        this.sanitizer = createInvocationSanitizer(config)
    }

    /**
     * Helper to only start the producer related code (e.g. when not a consumer)
     */
    public async startAsProducer() {
        if (this.kafkaProducer) {
            return
        }
        this.kafkaProducer = await KafkaProducerWrapper.create(this.kafkaClientRack, 'CDP_PRODUCER')
    }

    public async startAsConsumer(
        queue: CyclotronJobQueueKind,
        consumeBatch: (invocations: CyclotronJobInvocation[]) => Promise<{ backgroundTask: Promise<any> }>
    ) {
        this.queue = queue
        this.consumeBatch = consumeBatch

        const groupId = `cdp-cyclotron-${this.queue}-consumer`
        const topic = `cdp_cyclotron_${this.queue}`

        // NOTE: As there is only ever one consumer per process we use the KAFKA_CONSUMER_ vars as with any other consumer
        this.kafkaConsumer = createKafkaConsumer({ groupId, topic, callEachBatchWhenEmpty: true })

        logger.info('🔄', 'Connecting kafka consumer', { groupId, topic })
        await this.kafkaConsumer.connect(async (messages) => {
            const { backgroundTask } = await this.consumeKafkaBatch(messages)
            return { backgroundTask }
        })
    }

    public async stopConsumer() {
        await this.kafkaConsumer?.disconnect()
        this.kafkaConsumer = undefined
    }

    public async stopProducer() {
        await this.kafkaProducer?.disconnect()
        this.kafkaProducer = undefined
    }

    public isHealthy(): HealthCheckResult {
        if (!this.kafkaConsumer) {
            return new HealthCheckResultError('Kafka consumer not initialized', {})
        }
        return this.kafkaConsumer.isHealthy()
    }

    public async queueInvocations(invocations: CyclotronJobInvocation[]) {
        if (invocations.length === 0) {
            return
        }

        const producer = this.getKafkaProducer()

        // Pre-serialize all messages eagerly so the produce closures below only
        // capture lightweight strings instead of full invocation objects (globals, vmState, etc.)
        const messages = this.sanitizer.sanitizeInvocations(invocations).map((x) => {
            const jsonString = JSON.stringify(serializeInvocation(x))
            cdpJobSizeKb.labels('kafka').observe(jsonString.length / 1024)

            return {
                jsonString,
                queue: x.queue,
                id: x.id,
                functionId: x.functionId,
                teamId: x.teamId,
            }
        })

        await Promise.all(
            messages.map(async (msg) => {
                const value = this.config.CDP_CYCLOTRON_COMPRESS_KAFKA_DATA
                    ? await compress(msg.jsonString)
                    : msg.jsonString

                cdpJobSizeCompressedKb.labels('kafka').observe(value.length / 1024)

                const headers: Record<string, string> = {
                    // NOTE: Later we should remove hogFunctionId as it is no longer used
                    hogFunctionId: msg.functionId,
                    functionId: msg.functionId,
                    teamId: msg.teamId.toString(),
                }

                await producer
                    .produce({
                        value: Buffer.from(value),
                        key: Buffer.from(msg.id),
                        topic: `cdp_cyclotron_${msg.queue}`,
                        headers,
                    })
                    .catch((e) => {
                        logger.error('🔄', 'Error producing kafka message', {
                            error: String(e),
                            teamId: msg.teamId,
                            functionId: msg.functionId,
                            payloadSizeKb: value.length / 1024,
                        })

                        throw e
                    })
            })
        )
    }

    // Kafka jobs don't need explicit dequeue — they're just dropped
    public async dequeueInvocations(_invocations: CyclotronJobInvocation[]): Promise<void> {}
    public async heartbeatInvocations(_invocations: CyclotronJobInvocation[]): Promise<void> {}

    public async queueInvocationResults(invocationResults: CyclotronJobInvocationResult[]) {
        // With kafka we are essentially re-queuing the work to the target topic if it isn't finished
        const invocations: CyclotronJobInvocation[] = []
        for (const res of invocationResults) {
            if (!res.finished) {
                invocations.push(res.invocation)
            }
        }

        await this.queueInvocations(invocations)
    }

    private getKafkaProducer(): KafkaProducerWrapper {
        if (!this.kafkaProducer) {
            throw new Error('KafkaProducer not initialized')
        }
        return this.kafkaProducer
    }

    private async consumeKafkaBatch(messages: Message[]): Promise<{ backgroundTask: Promise<any> }> {
        if (messages.length === 0) {
            observeConsumedBatch({
                queue: this.queue!,
                source: 'kafka',
                batchSize: 0,
                maxBatchSize: this.consumerBatchSize,
            })
            return await this.consumeBatch!([])
        }

        const invocations: CyclotronJobInvocation[] = []

        for (const message of messages) {
            const rawValue = message.value
            if (!rawValue) {
                throw new Error('Bad message: ' + JSON.stringify(message))
            }

            // Try to decompress, otherwise just use the value as is
            const decompressedValue = await uncompress(rawValue).catch(() => rawValue)
            const invocation: CyclotronJobInvocation = migrateKafkaCyclotronInvocation(
                parseJSON(decompressedValue.toString())
            )

            invocation.queueSource = 'kafka' // NOTE: We always set this here, as we know it came from kafka
            invocations.push(invocation)
        }

        observeConsumedBatch({
            queue: this.queue!,
            source: 'kafka',
            batchSize: invocations.length,
            maxBatchSize: this.consumerBatchSize,
        })

        return await this.consumeBatch!(invocations)
    }
}

// NOTE: https://github.com/PostHog/posthog/pull/32588 modified the job format to move more things to the generic "state" value
// This function migrates any legacy jobs to the new format. We can remove this shortly after full release.
export function migrateKafkaCyclotronInvocation(invocation: CyclotronJobInvocation): CyclotronJobInvocation {
    // Type casting but keeping as a reference
    const unknownInvocation = invocation as Record<string, any>

    if ('hogFunctionId' in unknownInvocation) {
        // Must be the old format
        unknownInvocation.functionId = unknownInvocation.hogFunctionId
        unknownInvocation.state = {}
        delete unknownInvocation.hogFunctionId

        if ('vmState' in unknownInvocation) {
            unknownInvocation.state.vmState = unknownInvocation.vmState
            delete unknownInvocation.vmState
        }
        if ('globals' in unknownInvocation) {
            unknownInvocation.state.globals = unknownInvocation.globals
            delete unknownInvocation.globals
        }
        if ('timings' in unknownInvocation) {
            unknownInvocation.state.timings = unknownInvocation.timings
            delete unknownInvocation.timings
        }
    }

    return invocation
}

export function serializeInvocation(invocation: CyclotronJobInvocation): CyclotronJobInvocation {
    // NOTE: We are copying the object to ensure it is clean of any spare params
    return {
        id: invocation.id,
        teamId: invocation.teamId,
        functionId: invocation.functionId,
        parentRunId: invocation.parentRunId,
        state: invocation.state,
        queue: invocation.queue,
        queueParameters: invocation.queueParameters,
        queuePriority: invocation.queuePriority,
        queueScheduledAt: invocation.queueScheduledAt,
        queueMetadata: invocation.queueMetadata,
        queueSource: invocation.queueSource,
    }
}