:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { create } from '@bufbuild/protobuf'
import { Client, Code, ConnectError, createClient } from '@connectrpc/connect'
import { createGrpcTransport } from '@connectrpc/connect-node'
import { Counter } from 'prom-client'
import {
BillingUsageRecordSchema,
IngestBillingUsageRequestSchema,
UsageIngestion,
} from '~/common/generated/usage-ingestion/usage_ingestion/v1/service_pb'
import { logger } from '~/common/utils/logger'
import { delay } from '~/common/utils/utils'
const usageRecordsSentCounter = new Counter({
name: 'usage_ingestion_records_sent_total',
help: 'Usage records accepted by the usage-ingestion service.',
labelNames: ['producer_id', 'usage_key'],
})
const usageRecordsFailedCounter = new Counter({
name: 'usage_ingestion_records_failed_total',
help: 'Usage records dropped after the client exhausted its retries.',
labelNames: ['producer_id', 'usage_key', 'error_code'],
})
const usageRetriesCounter = new Counter({
name: 'usage_ingestion_retries_total',
help: 'Ingest calls retried after a transient error. A retry that succeeds drops no records.',
labelNames: ['producer_id', 'error_code'],
})
// Canceled and Internal are how connect-node reports a connection that went away underneath the
// call: the service ends every connection with GOAWAY at max_connection_age, and a flush that
// races that teardown never reaches the service. Retrying is safe because recordId makes ingest
// idempotent. Unlike personhog this set includes Canceled, because no caller passes an abort
// signal here, so Canceled can only mean the transport went away.
const RETRYABLE_CODES = new Set([
Code.Unavailable,
Code.DeadlineExceeded,
Code.ResourceExhausted,
Code.Aborted,
Code.Canceled,
Code.Internal,
])
const MAX_ATTEMPTS = 3
const RETRY_BACKOFF_MS = 100
/**
* The record IDs the service left out of its accepted list. The service skips a record it can
* never accept, such as one whose team has no organization, so a successful call can still
* drop records.
*/
export function rejectedRecords(chunk: UsageRecordInput[], acceptedRecordIds: string[]): Set<string> {
const accepted = new Set(acceptedRecordIds)
return new Set(chunk.filter((record) => !accepted.has(record.recordId)).map((record) => record.recordId))
}
/** The teams behind the rejected records, deduplicated so a repeated team reads as one lead. */
export function rejectedTeams(chunk: UsageRecordInput[], rejected: Set<string>): number[] {
const teams = new Set(chunk.filter((record) => rejected.has(record.recordId)).map((record) => record.teamId))
return [...teams].sort((a, b) => a - b)
}
export interface UsageRecordInput {
/** Unique per (teamId, producerId, usageKey). Reused verbatim on retry, which is what makes ingest idempotent. */
recordId: string
teamId: number
usageKey: string
unit: string
quantity: number
timestampMs: number
}
export interface UsageIngestionClientConfig {
addr: string
producerId: string
timeoutMs?: number
maxBatchSize?: number
useTls?: boolean
}
export class UsageIngestionClient {
private readonly client: Client<typeof UsageIngestion>
private readonly producerId: string
private readonly maxBatchSize: number
constructor(config: UsageIngestionClientConfig) {
const scheme = config.useTls ? 'https' : 'http'
this.client = createClient(
UsageIngestion,
createGrpcTransport({
baseUrl: `${scheme}://${config.addr}`,
defaultTimeoutMs: config.timeoutMs ?? 5_000,
})
)
this.producerId = config.producerId
this.maxBatchSize = config.maxBatchSize ?? 500
}
/** Returns the records that reached the service, rejected ones included: a retry fails them the same way. */
async ingest(records: UsageRecordInput[]): Promise<UsageRecordInput[]> {
if (records.length === 0) {
return []
}
const chunks: UsageRecordInput[][] = []
for (let i = 0; i < records.length; i += this.maxBatchSize) {
chunks.push(records.slice(i, i + this.maxBatchSize))
}
const received = await Promise.all(chunks.map((chunk) => this.ingestChunk(chunk)))
return chunks.filter((_, i) => received[i]).flat()
}
private async ingestChunk(chunk: UsageRecordInput[], attempt = 0): Promise<boolean> {
const request = create(IngestBillingUsageRequestSchema, {
records: chunk.map((record) =>
create(BillingUsageRecordSchema, {
recordId: record.recordId,
producerId: this.producerId,
teamId: BigInt(record.teamId),
usageKey: record.usageKey,
unit: record.unit,
quantity: BigInt(record.quantity),
timestampMs: BigInt(record.timestampMs),
})
),
})
try {
// The header names this producer in the service's own request metrics, which
// otherwise aggregate every caller into one series.
const response = await this.client.ingestBillingUsage(request, {
headers: { 'x-client-name': this.producerId },
})
const rejected = rejectedRecords(chunk, response.acceptedRecordIds)
for (const record of chunk) {
if (rejected.has(record.recordId)) {
continue
}
usageRecordsSentCounter.inc({ producer_id: this.producerId, usage_key: record.usageKey })
}
// The service drops records it cannot attribute rather than failing the batch, so a
// success can still leave some behind. Retrying them would fail the same way.
if (rejected.size > 0) {
for (const record of chunk) {
if (rejected.has(record.recordId)) {
usageRecordsFailedCounter.inc({
producer_id: this.producerId,
usage_key: record.usageKey,
error_code: 'rejected',
})
}
}
logger.warn('⚠️', 'usage-ingestion rejected usage records', {
producerId: this.producerId,
records: rejected.size,
teams: rejectedTeams(chunk, rejected),
})
}
return true
} catch (error) {
const code = error instanceof ConnectError ? error.code : Code.Unknown
if (attempt + 1 < MAX_ATTEMPTS && RETRYABLE_CODES.has(code)) {
usageRetriesCounter.inc({ producer_id: this.producerId, error_code: Code[code] ?? 'unknown' })
// Backoff so a burst of connection churn does not eat every attempt at once.
await delay(RETRY_BACKOFF_MS * 2 ** attempt)
return this.ingestChunk(chunk, attempt + 1)
}
for (const record of chunk) {
usageRecordsFailedCounter.inc({
producer_id: this.producerId,
usage_key: record.usageKey,
error_code: Code[code] ?? 'unknown',
})
}
logger.warn('⚠️', 'failed to ingest usage records', {
producerId: this.producerId,
records: chunk.length,
error: String(error),
})
return false
}
}
}