Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.4 KB
Raw Blame
import { create } from '@bufbuild/protobuf'
import { Client, Code, ConnectError, createClient } from '@connectrpc/connect'
import { createGrpcTransport } from '@connectrpc/connect-node'
import { Counter } from 'prom-client'

import {
    BillingUsageRecordSchema,
    IngestBillingUsageRequestSchema,
    UsageIngestion,
} from '~/common/generated/usage-ingestion/usage_ingestion/v1/service_pb'
import { logger } from '~/common/utils/logger'
import { delay } from '~/common/utils/utils'

const usageRecordsSentCounter = new Counter({
    name: 'usage_ingestion_records_sent_total',
    help: 'Usage records accepted by the usage-ingestion service.',
    labelNames: ['producer_id', 'usage_key'],
})

const usageRecordsFailedCounter = new Counter({
    name: 'usage_ingestion_records_failed_total',
    help: 'Usage records dropped after the client exhausted its retries.',
    labelNames: ['producer_id', 'usage_key', 'error_code'],
})

const usageRetriesCounter = new Counter({
    name: 'usage_ingestion_retries_total',
    help: 'Ingest calls retried after a transient error. A retry that succeeds drops no records.',
    labelNames: ['producer_id', 'error_code'],
})

// Canceled and Internal are how connect-node reports a connection that went away underneath the
// call: the service ends every connection with GOAWAY at max_connection_age, and a flush that
// races that teardown never reaches the service. Retrying is safe because recordId makes ingest
// idempotent. Unlike personhog this set includes Canceled, because no caller passes an abort
// signal here, so Canceled can only mean the transport went away.
const RETRYABLE_CODES = new Set([
    Code.Unavailable,
    Code.DeadlineExceeded,
    Code.ResourceExhausted,
    Code.Aborted,
    Code.Canceled,
    Code.Internal,
])

const MAX_ATTEMPTS = 3
const RETRY_BACKOFF_MS = 100

/**
 * The record IDs the service left out of its accepted list. The service skips a record it can
 * never accept, such as one whose team has no organization, so a successful call can still
 * drop records.
 */
export function rejectedRecords(chunk: UsageRecordInput[], acceptedRecordIds: string[]): Set<string> {
    const accepted = new Set(acceptedRecordIds)
    return new Set(chunk.filter((record) => !accepted.has(record.recordId)).map((record) => record.recordId))
}

/** The teams behind the rejected records, deduplicated so a repeated team reads as one lead. */
export function rejectedTeams(chunk: UsageRecordInput[], rejected: Set<string>): number[] {
    const teams = new Set(chunk.filter((record) => rejected.has(record.recordId)).map((record) => record.teamId))
    return [...teams].sort((a, b) => a - b)
}

export interface UsageRecordInput {
    /** Unique per (teamId, producerId, usageKey). Reused verbatim on retry, which is what makes ingest idempotent. */
    recordId: string
    teamId: number
    usageKey: string
    unit: string
    quantity: number
    timestampMs: number
}

export interface UsageIngestionClientConfig {
    addr: string
    producerId: string
    timeoutMs?: number
    maxBatchSize?: number
    useTls?: boolean
}

export class UsageIngestionClient {
    private readonly client: Client<typeof UsageIngestion>
    private readonly producerId: string
    private readonly maxBatchSize: number

    constructor(config: UsageIngestionClientConfig) {
        const scheme = config.useTls ? 'https' : 'http'
        this.client = createClient(
            UsageIngestion,
            createGrpcTransport({
                baseUrl: `${scheme}://${config.addr}`,
                defaultTimeoutMs: config.timeoutMs ?? 5_000,
            })
        )
        this.producerId = config.producerId
        this.maxBatchSize = config.maxBatchSize ?? 500
    }

    /** Returns the records that reached the service, rejected ones included: a retry fails them the same way. */
    async ingest(records: UsageRecordInput[]): Promise<UsageRecordInput[]> {
        if (records.length === 0) {
            return []
        }
        const chunks: UsageRecordInput[][] = []
        for (let i = 0; i < records.length; i += this.maxBatchSize) {
            chunks.push(records.slice(i, i + this.maxBatchSize))
        }
        const received = await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))
        return chunks.filter((_, i) => received[i]).flat()
    }

    private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<boolean> {
        const request = create(IngestBillingUsageRequestSchema, {
            records: chunk.map((record) =>
                create(BillingUsageRecordSchema, {
                    recordId: record.recordId,
                    producerId: this.producerId,
                    teamId: BigInt(record.teamId),
                    usageKey: record.usageKey,
                    unit: record.unit,
                    quantity: BigInt(record.quantity),
                    timestampMs: BigInt(record.timestampMs),
                })
            ),
        })

        try {
            // The header names this producer in the service's own request metrics, which
            // otherwise aggregate every caller into one series.
            const response = await this.client.ingestBillingUsage(request, {
                headers: { 'x-client-name': this.producerId },
            })
            const rejected = rejectedRecords(chunk, response.acceptedRecordIds)
            for (const record of chunk) {
                if (rejected.has(record.recordId)) {
                    continue
                }
                usageRecordsSentCounter.inc({ producer_id: this.producerId, usage_key: record.usageKey })
            }
            // The service drops records it cannot attribute rather than failing the batch, so a
            // success can still leave some behind. Retrying them would fail the same way.
            if (rejected.size > 0) {
                for (const record of chunk) {
                    if (rejected.has(record.recordId)) {
                        usageRecordsFailedCounter.inc({
                            producer_id: this.producerId,
                            usage_key: record.usageKey,
                            error_code: 'rejected',
                        })
                    }
                }
                logger.warn('⚠️', 'usage-ingestion rejected usage records', {
                    producerId: this.producerId,
                    records: rejected.size,
                    teams: rejectedTeams(chunk, rejected),
                })
            }
            return true
        } catch (error) {
            const code = error instanceof ConnectError ? error.code : Code.Unknown
            if (attempt + 1 < MAX_ATTEMPTS && RETRYABLE_CODES.has(code)) {
                usageRetriesCounter.inc({ producer_id: this.producerId, error_code: Code[code] ?? 'unknown' })
                // Backoff so a burst of connection churn does not eat every attempt at once.
                await delay(RETRY_BACKOFF_MS * 2 ** attempt)
                return this.ingestChunk(chunk, attempt + 1)
            }
            for (const record of chunk) {
                usageRecordsFailedCounter.inc({
                    producer_id: this.producerId,
                    usage_key: record.usageKey,
                    error_code: Code[code] ?? 'unknown',
                })
            }
            logger.warn('⚠️', 'failed to ingest usage records', {
                producerId: this.producerId,
                records: chunk.length,
                error: String(error),
            })
            return false
        }
    }
}