Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.1 KB
Raw Blame
import { trace } from '@opentelemetry/api'
import { Counter } from 'prom-client'

import { type RedisV2 } from '~/common/redis/redis-v2'
import { KeyedRateLimitRequest, KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { instrumented } from '~/common/tracing/tracing-utils'
import { logger } from '~/common/utils/logger'
import { type LogRecord } from '~/logs/log-record-avro'
import { type BatchContext, type FilterResult, type PipelineStage } from '~/logs/pipeline/log-processing-pipeline'

import type { CompiledRuleSet, EvaluateResult, RateLimitPendingByRule, SamplingClassifyResult } from './evaluate'
import {
    SAMPLING_DECISION_DROP,
    SAMPLING_DECISION_KEEP,
    SAMPLING_DECISION_SAMPLE_DROPPED,
    classifySamplingRecord,
    evaluateLogRecord,
} from './evaluate'

/**
 * Incremented when a per-record sampling evaluation throws. The record is kept
 * (fail-open) so an adversarial or malformed rule cannot drop traffic via an
 * uncaught exception, but the counter makes the silent failure observable.
 */
export const logsSamplingEvalErrorCounter = new Counter({
    name: 'logs_ingestion_sampling_eval_error_total',
    help: 'Per-record sampling evaluation threw an exception; record was kept (fail-open).',
    labelNames: ['team_id', 'phase'],
})

function safeClassifySamplingRecord(
    ruleSet: CompiledRuleSet,
    record: LogRecord,
    teamId: number
): SamplingClassifyResult {
    try {
        return classifySamplingRecord(ruleSet, record)
    } catch (err) {
        logsSamplingEvalErrorCounter.inc({ team_id: String(teamId), phase: 'classify' })
        logger.warn('[logs-sampling] classifySamplingRecord threw — keeping record', { teamId, err })
        return { kind: 'resolved', decision: SAMPLING_DECISION_KEEP, ruleId: null }
    }
}

function safeEvaluateLogRecord(ruleSet: CompiledRuleSet, record: LogRecord, teamId: number): EvaluateResult {
    try {
        return evaluateLogRecord(ruleSet, record)
    } catch (err) {
        logsSamplingEvalErrorCounter.inc({ team_id: String(teamId), phase: 'evaluate' })
        logger.warn('[logs-sampling] evaluateLogRecord threw — keeping record', { teamId, err })
        return { decision: SAMPLING_DECISION_KEEP, ruleId: null }
    }
}

const recordBytes = (r: LogRecord): number => r.bytes_uncompressed ?? 0

export const SAMPLING_STAGE_NAME = 'sampling'

export class LogsSamplingService {
    private rateLimiter: KeyedRateLimiterService

    constructor(redis: RedisV2, ttlSeconds: number) {
        this.rateLimiter = new KeyedRateLimiterService(
            { name: 'logs-sampling-rate', ttlSeconds, scriptVersion: 'v3' },
            redis
        )
    }

    /**
     * Applies drop rules + rate limits to already-decoded records, returning the survivors and the
     * per-rule drop accounting. Decode/encode, PII scrub, metric extraction, and hog transforms are
     * owned by the pipeline (`processLogMessageBuffer`); this is the body of the sampling `filter`
     * stage (see `makeSamplingStage`).
     */
    @instrumented({
        key: 'logsIngestion.sampling.sampleRecords',
        measureTime: false,
        sendException: false,
    })
    public async sampleRecords(
        records: LogRecord[],
        ruleSet: CompiledRuleSet,
        batch: BatchContext,
        teamId?: number,
        headerBytesUncompressed: number = 0
    ): Promise<FilterResult> {
        const kept: LogRecord[] = []
        let recordsDropped = 0
        const recordsDroppedByRuleId = new Map<string, number>()
        let bytesDropped = 0
        const bytesDroppedByRuleId = new Map<string, number>()
        let contentBytesDropped = 0

        const useRate = Boolean(ruleSet.hasRateLimitRules && teamId != null)

        if (useRate && teamId != null) {
            const classifications = records.map((r) => safeClassifySamplingRecord(ruleSet, r, teamId))
            const pendingByRule: RateLimitPendingByRule = new Map()
            for (let i = 0; i < records.length; i++) {
                const c = classifications[i]!
                if (c.kind === 'rate_limit') {
                    const list = pendingByRule.get(c.ruleId) ?? []
                    list.push(i)
                    pendingByRule.set(c.ruleId, list)
                }
            }
            const rateKeep = await this.applyRateLimits(
                teamId,
                ruleSet,
                pendingByRule,
                records,
                headerBytesUncompressed,
                batch
            )

            for (let i = 0; i < records.length; i++) {
                const record = records[i]!
                const c = classifications[i]!
                if (c.kind === 'rate_limit') {
                    if (rateKeep.get(i) === false) {
                        const rb = recordBytes(record)
                        recordsDropped++
                        bytesDropped += rb
                        contentBytesDropped += batch.contentBytesOf(record)
                        recordsDroppedByRuleId.set(c.ruleId, (recordsDroppedByRuleId.get(c.ruleId) ?? 0) + 1)
                        bytesDroppedByRuleId.set(c.ruleId, (bytesDroppedByRuleId.get(c.ruleId) ?? 0) + rb)
                        continue
                    }
                    kept.push(record)
                    continue
                }
                if (c.decision === SAMPLING_DECISION_DROP || c.decision === SAMPLING_DECISION_SAMPLE_DROPPED) {
                    const rb = recordBytes(record)
                    recordsDropped++
                    bytesDropped += rb
                    contentBytesDropped += batch.contentBytesOf(record)
                    if (c.ruleId != null) {
                        recordsDroppedByRuleId.set(c.ruleId, (recordsDroppedByRuleId.get(c.ruleId) ?? 0) + 1)
                        bytesDroppedByRuleId.set(c.ruleId, (bytesDroppedByRuleId.get(c.ruleId) ?? 0) + rb)
                    }
                    continue
                }
                kept.push(record)
            }
        } else {
            for (const record of records) {
                const { decision, ruleId } = safeEvaluateLogRecord(ruleSet, record, teamId ?? 0)
                if (decision === SAMPLING_DECISION_DROP || decision === SAMPLING_DECISION_SAMPLE_DROPPED) {
                    const rb = recordBytes(record)
                    recordsDropped++
                    bytesDropped += rb
                    contentBytesDropped += batch.contentBytesOf(record)
                    if (ruleId != null) {
                        recordsDroppedByRuleId.set(ruleId, (recordsDroppedByRuleId.get(ruleId) ?? 0) + 1)
                        bytesDroppedByRuleId.set(ruleId, (bytesDroppedByRuleId.get(ruleId) ?? 0) + rb)
                    }
                    continue
                }
                kept.push(record)
            }
        }

        trace.getActiveSpan()?.setAttributes({
            'logs.sampling.input_record_count': records.length,
            'logs.sampling.kept_record_count': kept.length,
            'logs.sampling.dropped_record_count': recordsDropped,
            'logs.sampling.all_dropped': kept.length === 0,
        })

        return {
            kept,
            stats: {
                recordsDropped,
                recordsDroppedByRuleId,
                bytesDropped,
                bytesDroppedByRuleId,
                contentBytesDropped,
            },
        }
    }

    /** The sampling drop-rules + rate-limit `filter` stage for the log processing pipeline. */
    public makeSamplingStage(
        ruleSet: CompiledRuleSet,
        teamId?: number,
        headerBytesUncompressed: number = 0
    ): PipelineStage {
        return {
            kind: 'filter',
            name: SAMPLING_STAGE_NAME,
            run: (records, batch) => this.sampleRecords(records, ruleSet, batch, teamId, headerBytesUncompressed),
        }
    }

    /**
     * Maps a recordIndex -> keep (true) or drop (false) decision per rate_limit rule.
     * Each rule's pending lines share one Lua call; lines are admitted while their
     * accumulated cost stays within the pre-batch token budget. Cost is one token
     * per record (`costUnit: 'records'`) or, for `costUnit: 'bytes'`, each row's
     * pro-rata share of the batch header `bytesUncompressed`
     * (`headerBytesUncompressed × contentBytesOf(row) / contentBytesTotal`) — the same
     * unit billing meters, so the limiter admits at the configured byte rate instead
     * of over-counting the per-row `bytes_uncompressed` (which re-includes shared
     * batch data on every row). Falls back to per-row `bytes_uncompressed` when the
     * header is unavailable (older producers).
     */
    private async applyRateLimits(
        teamId: number,
        ruleSet: CompiledRuleSet,
        pendingByRule: RateLimitPendingByRule,
        records: LogRecord[],
        headerBytesUncompressed: number,
        batch: BatchContext
    ): Promise<Map<number, boolean>> {
        const keepByIndex = new Map<number, boolean>()
        if (pendingByRule.size === 0) {
            return keepByIndex
        }

        const proRataScale =
            headerBytesUncompressed > 0 && batch.contentBytesTotal > 0
                ? headerBytesUncompressed / batch.contentBytesTotal
                : 0
        const byteCost = (idx: number): number =>
            proRataScale > 0 ? batch.contentBytesOf(records[idx]!) * proRataScale : recordBytes(records[idx]!)

        const ruleById = new Map(ruleSet.rules.map((r) => [r.id, r]))
        type Entry = { indices: number[]; costs: number[]; req: KeyedRateLimitRequest }
        const entries: Entry[] = []

        for (const [ruleId, indices] of pendingByRule) {
            const rl = ruleById.get(ruleId)?.rateLimit
            if (!rl || indices.length === 0) {
                continue
            }
            const costs = rl.costUnit === 'bytes' ? indices.map(byteCost) : indices.map(() => 1)
            const totalCost = costs.reduce((a, b) => a + b, 0)
            entries.push({
                indices,
                costs,
                req: {
                    id: `${teamId}/${ruleId}`,
                    cost: totalCost,
                    bucketSize: rl.poolMax,
                    refillRate: rl.refillPerSecond,
                },
            })
        }

        if (entries.length === 0) {
            for (const [, indices] of pendingByRule) {
                for (const idx of indices) {
                    keepByIndex.set(idx, true)
                }
            }
            return keepByIndex
        }

        const results = await this.rateLimiter.rateLimitMany(entries.map((e) => e.req))

        for (let i = 0; i < entries.length; i++) {
            const { indices, costs } = entries[i]!
            const tokensBefore = results[i]?.[1].tokensBefore ?? 0
            const budget = Math.max(0, Math.floor(tokensBefore))
            let spent = 0
            for (let j = 0; j < indices.length; j++) {
                const next = spent + costs[j]!
                const admit = next <= budget
                keepByIndex.set(indices[j]!, admit)
                if (admit) {
                    spent = next
                }
            }
        }

        return keepByIndex
    }
}