Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.9 KB
Raw Blame
import { DateTime } from 'luxon'
import { Counter } from 'prom-client'

import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { safeClickhouseString } from '~/common/utils/db/utils'
import { castTimestampOrNow, castTimestampToClickhouseFormat } from '~/common/utils/utils'

import { ClickHouseTimestamp, TimestampFormat } from '../../types'

const appMetricsAggregatorQueuedCounter = new Counter({
    name: 'app_metrics_aggregator_queued_total',
    help: 'App metric items queued — counted before in-memory dedup.',
    labelNames: ['app_source'],
})

const appMetricsAggregatorFlushedCounter = new Counter({
    name: 'app_metrics_aggregator_flushed_total',
    help: 'Unique app metric rows produced to Kafka after in-memory dedup. Dedup rate = 1 - (flushed / queued).',
    labelNames: ['app_source'],
})

/**
 * One v2 app metric row, matching the ClickHouse `app_metrics2` schema.
 * Aggregation key is the six identity fields plus the timestamp hour — entries
 * sharing them have their `count` summed in-memory and emitted as one Kafka
 * message on `flush`.
 */
export interface AppMetricInput {
    team_id: number
    app_source: string
    app_source_id: string
    instance_id?: string
    metric_kind: string
    metric_name: string
    count: number
    /**
     * Defaults to the flush time. Set it to date the row at another hour, e.g. a future
     * month of logs retention that should bill when that month starts. Truncated to the hour,
     * the same grain `app_metrics2` aggregates at.
     */
    timestamp?: DateTime
}

type BufferedAppMetric = Omit<AppMetricInput, 'instance_id' | 'timestamp'> & {
    instance_id: string
    timestamp: ClickHouseTimestamp | null
}

/**
 * Dedupes v2 app metrics on `queue`, produces them on `flush`.
 *
 * The Kafka routing decision (which producer, which topic) is injected as an
 * `IngestionOutputs` registry that must include `APP_METRICS_OUTPUT`. The
 * aggregator pulls the resolved output by name on `flush`. No background
 * flushing, no lifecycle — caller is expected to `flush()` at the end of
 * whatever batch / cycle they want metrics emitted for.
 */
export class AppMetricsAggregator {
    private buffer = new Map<string, BufferedAppMetric>()

    constructor(private readonly outputs: IngestionOutputs<AppMetricsOutput>) {}

    queue(metric: AppMetricInput): void {
        appMetricsAggregatorQueuedCounter.inc({ app_source: metric.app_source })
        const { timestamp, ...rest } = metric
        const row: BufferedAppMetric = {
            ...rest,
            instance_id: metric.instance_id ?? '',
            timestamp: timestamp
                ? castTimestampToClickhouseFormat(timestamp.toUTC().startOf('hour'), TimestampFormat.ClickHouse)
                : null,
        }
        const key = makeKey(row)
        const existing = this.buffer.get(key)
        if (existing) {
            existing.count += metric.count
        } else {
            this.buffer.set(key, row)
        }
    }

    async flush(): Promise<void> {
        if (this.buffer.size === 0) {
            return
        }
        const drained = [...this.buffer.values()]
        this.buffer.clear()

        const flushTimestamp = castTimestampOrNow(null, TimestampFormat.ClickHouse)
        // No partition key — rows are re-aggregated by ClickHouse, ordering is
        // irrelevant, and round-robin distributes load evenly across partitions.
        const messages = drained.map((m) => ({
            value: Buffer.from(
                // safeClickhouseString strips lone Unicode surrogates that ClickHouse rejects —
                // identity fields are PostHog-generated UUIDs today, but cheap insurance if
                // any of them ever take user-controlled values.
                safeClickhouseString(
                    JSON.stringify({
                        team_id: m.team_id,
                        timestamp: m.timestamp ?? flushTimestamp,
                        app_source: m.app_source,
                        app_source_id: m.app_source_id,
                        instance_id: m.instance_id,
                        metric_kind: m.metric_kind,
                        metric_name: m.metric_name,
                        count: m.count,
                    })
                )
            ),
            key: null,
        }))
        await this.outputs.queueMessages(APP_METRICS_OUTPUT, messages)

        // Increment after the await so a failed produce isn't counted as flushed —
        // dedup-rate = 1 - (flushed / queued) stays meaningful in error scenarios.
        for (const m of drained) {
            appMetricsAggregatorFlushedCounter.inc({ app_source: m.app_source })
        }
    }
}

function makeKey(m: BufferedAppMetric): string {
    return `${m.team_id}:${m.app_source}:${m.app_source_id}:${m.instance_id}:${m.metric_kind}:${m.metric_name}:${m.timestamp ?? ''}`
}