Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.9 KB
Raw Blame
import { Histogram } from 'prom-client'

import { RedisV2 } from '~/common/redis/redis-v2'
import { KeyedRateLimitRequest, KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { LogsIngestionConsumerConfig } from '~/logs/config'
import { LogsIngestionMessage } from '~/logs/types'

/** Convert milliseconds to seconds */
const msToSeconds = (ms: number): number => Math.round(ms / 1000)

/** Convert bytes to kilobytes (rounded up) */
const bytesToKb = (bytes: number): number => Math.ceil(bytes / 1000)

export const logsMessageLagHistogram = new Histogram({
    name: 'logs_rate_limiter_message_lag_seconds',
    help: 'Lag between message observed timestamp and wall clock time (seconds)',
    buckets: [-60, 0, 1, 5, 10, 30, 60, 300, 600, 1800, 3600],
})

export type LogsRateLimiterConfig = Pick<
    LogsIngestionConsumerConfig,
    | 'LOGS_LIMITER_TEAM_BUCKET_SIZE_KB'
    | 'LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND'
    | 'LOGS_LIMITER_DISABLED_FOR_TEAMS'
    | 'LOGS_LIMITER_ENABLED_TEAMS'
    | 'LOGS_LIMITER_BUCKET_SIZE_KB'
    | 'LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND'
    | 'LOGS_LIMITER_TTL_SECONDS'
>

const DEFAULT_LIMITER_NAME = 'logs-rate-limiter'

/** Redis key prefix for a given limiter name. Each ingestion type (logs, traces) gets its own
 * namespace so their token buckets don't share — and deplete — the same per-team state. */
export const buildBaseRedisKey = (limiterName: string = DEFAULT_LIMITER_NAME): string =>
    process.env.NODE_ENV == 'test' ? `@posthog-test/${limiterName}` : `@posthog/${limiterName}`

/** Re-exported for the consumer test which needs to clean up keys between runs. */
export const BASE_REDIS_KEY = buildBaseRedisKey()

export type LogsRateLimit = {
    tokensBefore: number
    tokensAfter: number
    isRateLimited: boolean
}

export type FilteredMessages = {
    allowed: LogsIngestionMessage[]
    dropped: LogsIngestionMessage[]
}

/**
 * The LogsRateLimiterService is used to rate limit logs ingestion to ensure we aren't allowing too many logs to be ingested at once.
 * The key part is we specify the refill rate as our per second KB/s limit. and the bucket size as the amount we are allowed to burst to.
 * The burst shouldn't be too much higher.
 */
export class LogsRateLimiterService {
    private teamBucketSizes: Map<number, number>
    private teamRefillRates: Map<number, number>
    private disabledTeamIds: Set<number> | '*' | null
    private enabledTeamIds: Set<number> | '*' | null
    private rateLimiter: KeyedRateLimiterService

    constructor(
        private config: LogsRateLimiterConfig,
        redis: RedisV2,
        limiterName: string = DEFAULT_LIMITER_NAME
    ) {
        this.teamBucketSizes = this.parseTeamConfig(config.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB)
        this.teamRefillRates = this.parseTeamConfig(config.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND)
        this.disabledTeamIds = this.parseTeamIdList(config.LOGS_LIMITER_DISABLED_FOR_TEAMS)
        this.enabledTeamIds = this.parseTeamIdList(config.LOGS_LIMITER_ENABLED_TEAMS)
        this.rateLimiter = new KeyedRateLimiterService(
            // Bucket params are passed per-request so live `this.config` mutations
            // (used in tests) take effect. failOpen=false preserves the legacy
            // throw-on-Redis-outage behaviour.
            { name: limiterName, failOpen: false },
            redis
        )
    }

    private parseTeamIdList(config: string): Set<number> | '*' | null {
        if (config === '*') {
            return '*'
        }
        if (!config) {
            return null
        }
        const ids = new Set<number>()
        for (const id of config.split(',')) {
            const parsed = parseInt(id.trim(), 10)
            if (!isNaN(parsed)) {
                ids.add(parsed)
            }
        }
        return ids
    }

    private parseTeamConfig(config: string): Map<number, number> {
        const result = new Map<number, number>()
        if (!config) {
            return result
        }
        for (const entry of config.split(',')) {
            const [teamId, value] = entry.split(':').map((s) => parseInt(s.trim(), 10))
            if (!isNaN(teamId) && !isNaN(value)) {
                result.set(teamId, value)
            }
        }
        return result
    }

    private getHeaderValue(headers: any[] | undefined, key: string): string | undefined {
        if (!headers || !Array.isArray(headers)) {
            return undefined
        }

        for (const header of headers) {
            if (header[key]) {
                // Convert Buffer to string
                return Buffer.from(header[key]).toString('utf8')
            }
        }
        return undefined
    }

    private getTimestampFromMessage(message: LogsIngestionMessage): number {
        const createdAtHeader = this.getHeaderValue(message.message.headers, 'created_at')
        if (createdAtHeader) {
            const timestamp = Date.parse(createdAtHeader) // Parse RFC3339/ISO8601 string
            if (!isNaN(timestamp)) {
                return msToSeconds(timestamp)
            }
        }
        // Fallback to current time
        return msToSeconds(Date.now())
    }

    public async rateLimitMany(idCosts: [string, number, number][]): Promise<[string, LogsRateLimit][]> {
        const requests: KeyedRateLimitRequest[] = idCosts.map(([id, cost, nowSeconds]) => {
            const teamId = parseInt(id, 10)
            return {
                id,
                cost,
                now: nowSeconds,
                bucketSize: this.teamBucketSizes.get(teamId) ?? this.config.LOGS_LIMITER_BUCKET_SIZE_KB,
                refillRate: this.teamRefillRates.get(teamId) ?? this.config.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND,
                ttlSeconds: this.config.LOGS_LIMITER_TTL_SECONDS,
            }
        })
        const results = await this.rateLimiter.rateLimitMany(requests)
        return results.map(([id, r]) => [
            id,
            { tokensBefore: r.tokensBefore, tokensAfter: r.tokens, isRateLimited: r.isRateLimited },
        ])
    }

    private isRateLimitingEnabledForTeam(teamId: number): boolean {
        if (this.disabledTeamIds === '*') {
            return false
        }

        if (this.disabledTeamIds?.has(teamId)) {
            return false
        }

        if (this.enabledTeamIds === '*') {
            return true
        }

        if (!this.enabledTeamIds) {
            return false
        }

        return this.enabledTeamIds.has(teamId)
    }

    public async filterMessages(messages: LogsIngestionMessage[]): Promise<FilteredMessages> {
        // Group messages by team to calculate total cost per team (only for teams with rate limiting enabled)
        const teamCosts = new Map<number, number>()
        const teamOldestTimestamps = new Map<number, number>()
        const teamNewestTimestamps = new Map<number, number>()

        for (const message of messages) {
            const messageTimestamp = this.getTimestampFromMessage(message)

            const existingOldest = teamOldestTimestamps.get(message.teamId)
            if (existingOldest === undefined || messageTimestamp < existingOldest) {
                teamOldestTimestamps.set(message.teamId, messageTimestamp)
            }

            const existingNewest = teamNewestTimestamps.get(message.teamId)
            if (existingNewest === undefined || messageTimestamp > existingNewest) {
                teamNewestTimestamps.set(message.teamId, messageTimestamp)
            }

            if (!this.isRateLimitingEnabledForTeam(message.teamId)) {
                continue
            }
            const currentCost = teamCosts.get(message.teamId) ?? 0
            // Cost is in KB (uncompressed bytes / 1000)
            const costKb = bytesToKb(message.bytesUncompressed)
            teamCosts.set(message.teamId, currentCost + costKb)
        }

        // Track how far behind wall clock the messages are (using oldest per team for worst-case lag)
        const nowSeconds = msToSeconds(Date.now())
        for (const [, timestamp] of teamOldestTimestamps) {
            logsMessageLagHistogram.observe(nowSeconds - timestamp)
        }

        // Use the newest timestamp per team for the token bucket: this gives the bucket credit
        // for the full producer-time span the batch covers, so a catch-up batch refills against
        // its own time range instead of seeing timeDiff=0 from the previous batch's last call.
        const rateLimitResults = await this.rateLimitMany(
            Array.from(teamCosts.entries()).map(([teamId, cost]) => [
                teamId.toString(),
                cost,
                teamNewestTimestamps.get(teamId) ?? msToSeconds(Date.now()),
            ])
        )

        // Build a map of team rate limit results
        const teamLimits = new Map<number, { tokensBefore: number; tokensAfter: number; isRateLimited: boolean }>()
        for (const [teamIdStr, result] of rateLimitResults) {
            teamLimits.set(parseInt(teamIdStr, 10), result)
        }

        // Filter messages based on rate limits, allowing partial batches through
        const allowed: LogsIngestionMessage[] = []
        const dropped: LogsIngestionMessage[] = []
        const teamKbUsed = new Map<number, number>()

        for (const message of messages) {
            const limit = teamLimits.get(message.teamId)
            if (!limit) {
                // No rate limit for this team (either not enabled or not in the map)
                allowed.push(message)
                continue
            }

            const kbUsed = teamKbUsed.get(message.teamId) ?? 0
            const availableKb = limit.tokensBefore
            const messageKb = bytesToKb(message.bytesUncompressed)

            // Allow message if we haven't exceeded the available tokens
            if (kbUsed + messageKb <= availableKb) {
                allowed.push(message)
                teamKbUsed.set(message.teamId, kbUsed + messageKb)
            } else {
                dropped.push(message)
            }
        }

        return { allowed, dropped }
    }
}