Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

3.4 KB
Raw Blame
import { PromiseScheduler } from '~/common/utils/promise-scheduler'
import { CompletedSubBatch, MessageCoordinates, StreamIngestDriver, SubBatchBudget } from '~/ingestion/api/grpc-server'
import { deserializeKafkaMessage } from '~/ingestion/api/kafka-message-converter'
import { SerializedKafkaMessage } from '~/ingestion/api/types'
import { BatchBudget, budgetDeadline } from '~/ingestion/framework/batch-budget'
import { FeedResult } from '~/ingestion/framework/batching-pipeline'
import { createKafkaDebugContext, createOkContext } from '~/ingestion/framework/helpers'
import { isTimeoutResult } from '~/ingestion/framework/results'
import {
    JoinedIngestionPipelineContext,
    JoinedIngestionPipelineInput,
    createJoinedIngestionPipeline,
} from '~/ingestion/pipelines/analytics/joined-ingestion-pipeline'

/** Batch context fed with each gRPC sub-batch so its completion routes back to the right stream. */
export type GrpcBatchContext = { grpcStreamId: number; grpcSeq: number }

export type GrpcIngestPipeline = ReturnType<
    typeof createJoinedIngestionPipeline<JoinedIngestionPipelineInput, JoinedIngestionPipelineContext, GrpcBatchContext>
>

/**
 * Pipeline mechanics for the gRPC stream server. The `settled` promise on
 * each completed batch is the ack barrier: it mirrors the HTTP handler's
 * contract (side effects plus the promise scheduler) but stays a promise
 * so the server can settle many batches concurrently.
 */
export class GrpcStreamIngestDriver implements StreamIngestDriver {
    constructor(
        private pipeline: GrpcIngestPipeline,
        private promiseScheduler: PromiseScheduler
    ) {}

    feed(
        streamId: number,
        seq: number,
        messages: SerializedKafkaMessage[],
        budget: SubBatchBudget
    ): Promise<FeedResult> {
        const batch = messages.map((serialized) => {
            const message = deserializeKafkaMessage(serialized)
            return createOkContext({ message }, { message, debugContext: createKafkaDebugContext(message) })
        })
        return this.pipeline.feed(batch, { grpcStreamId: streamId, grpcSeq: seq }, this.wireBudget(budget))
    }

    private wireBudget({ armedAt, softBudgetMs }: SubBatchBudget): BatchBudget {
        const softAt = budgetDeadline(armedAt, softBudgetMs)
        return softAt === null ? BatchBudget.unlimited() : BatchBudget.softDeadline(softAt)
    }

    async next(): Promise<CompletedSubBatch | null> {
        const result = await this.pipeline.next()
        if (result === null) {
            return null
        }
        // waitForAll snapshots the currently scheduled promises, so this
        // settles even under sustained load from other batches.
        const settled = (async (): Promise<void> => {
            await Promise.all(result.sideEffects ?? [])
            await this.promiseScheduler.waitForAll()
        })()
        const unfinished: MessageCoordinates[] = []
        for (const { result: elementResult, context } of result.elements) {
            if (isTimeoutResult(elementResult)) {
                unfinished.push({ partition: context.message.partition, offset: context.message.offset })
            }
        }
        return {
            streamId: result.batchContext.grpcStreamId,
            seq: result.batchContext.grpcSeq,
            accepted: result.elements.length - unfinished.length,
            unfinished,
            settled,
        }
    }
}