Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.6 KB
Raw Blame
import { ValueMatcher } from '~/types'

import { UsageIngestionClient, UsageRecordInput } from './client'

export interface UsageRecordBatchConfig {
    unit: string
    isTeamEnabled: ValueMatcher<number>
}

interface PendingRecord {
    teamId: number
    usageKey: string
    recordId: string
    quantity: number
    unit?: string
    timestampMs?: number
}

/**
 * Collects one usage record per billed thing, identified by a record ID the
 * producer can reproduce. Every record carries quantity 1, so a second `add` of
 * the same identity is the same thing seen twice and is discarded rather than
 * added — the aggregate lives in ClickHouse, keyed the same way.
 *
 * Keyed by (team, usage key, record ID) to match the ingest contract's identity.
 * Keying on the record ID alone would merge two teams that share an ID and bill
 * both to the first one.
 */
export class UsageRecordBatch {
    private records = new Map<string, PendingRecord>()
    private pendingAcknowledgements: Promise<void>[] = []

    constructor(
        private readonly client: UsageIngestionClient | null,
        private readonly config: UsageRecordBatchConfig
    ) {}

    get size(): number {
        return this.records.size
    }

    /** Whether this team's records would be kept, so a caller can skip building one. */
    accepts(teamId: number): boolean {
        return this.client !== null && this.config.isTeamEnabled(teamId)
    }

    add(teamId: number, usageKey: string, recordId: string, quantity = 1, unit?: string, timestampMs?: number): void {
        if (quantity <= 0 || !this.accepts(teamId)) {
            return
        }
        const key = `${teamId}:${usageKey}:${recordId}`
        if (!this.records.has(key)) {
            this.records.set(key, { teamId, usageKey, recordId, quantity, unit, timestampMs })
        }
    }

    /**
     * Adds a record only after every Kafka write for its logical event has
     * succeeded. The flush side effect waits for these acknowledgements, so it
     * is safe to schedule both production and usage reporting in the background.
     */
    addAfterAcknowledgements(
        acknowledgements: Promise<unknown | null>[],
        teamId: number,
        usageKey: string,
        recordId: string,
        timestampMs?: number
    ): void {
        // A record that would be dropped must not put the flush behind its Kafka writes.
        if (!this.accepts(teamId)) {
            return
        }
        this.pendingAcknowledgements.push(
            Promise.all(acknowledgements)
                .then((results) => {
                    if (results.every((result) => result !== null)) {
                        this.add(teamId, usageKey, recordId, 1, undefined, timestampMs)
                    }
                })
                // Kafka errors are handled by the producer side effect. They must
                // not turn into a billing record or an unhandled rejection here.
                .catch(() => undefined)
        )
    }

    /**
     * Sends what is queued when the call starts. An acknowledgement that lands while this
     * awaits belongs to the next flush, so a caller that keeps adding stays bounded: one
     * pass, one send. A caller that is finished adding wants {@link drain} instead, because
     * nothing would flush that record afterwards.
     */
    async flush(): Promise<UsageRecordInput[]> {
        const pending = this.pendingAcknowledgements
        this.pendingAcknowledgements = []
        await Promise.all(pending)
        if (!this.client || this.records.size === 0) {
            return []
        }
        // Aggregate producers have no per-record capture time, so retain their flush-time clock.
        const flushedAtMs = Date.now()
        const records: UsageRecordInput[] = [...this.records.values()].map((record) => ({
            recordId: record.recordId,
            teamId: record.teamId,
            usageKey: record.usageKey,
            unit: record.unit ?? this.config.unit,
            quantity: record.quantity,
            timestampMs: record.timestampMs ?? flushedAtMs,
        }))
        this.records.clear()
        return this.client.ingest(records)
    }

    /**
     * Flushes until nothing is left queued. Only for a caller that has stopped adding: the
     * end of a batch, or a consumer shutting down. Records live in memory until they are
     * sent, so a batch that ends without draining bills nothing for the writes it was still
     * waiting on.
     */
    async drain(): Promise<void> {
        do {
            await this.flush()
        } while (this.pendingAcknowledgements.length > 0 || this.records.size > 0)
    }
}