:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Histogram } from 'prom-client'
import { RedisV2 } from '~/common/redis/redis-v2'
import { KeyedRateLimitRequest, KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { LogsIngestionConsumerConfig } from '~/logs/config'
import { LogsIngestionMessage } from '~/logs/types'
/** Convert milliseconds to seconds */
const msToSeconds = (ms: number): number => Math.round(ms / 1000)
/** Convert bytes to kilobytes (rounded up) */
const bytesToKb = (bytes: number): number => Math.ceil(bytes / 1000)
export const logsMessageLagHistogram = new Histogram({
name: 'logs_rate_limiter_message_lag_seconds',
help: 'Lag between message observed timestamp and wall clock time (seconds)',
buckets: [-60, 0, 1, 5, 10, 30, 60, 300, 600, 1800, 3600],
})
export type LogsRateLimiterConfig = Pick<
LogsIngestionConsumerConfig,
| 'LOGS_LIMITER_TEAM_BUCKET_SIZE_KB'
| 'LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND'
| 'LOGS_LIMITER_DISABLED_FOR_TEAMS'
| 'LOGS_LIMITER_ENABLED_TEAMS'
| 'LOGS_LIMITER_BUCKET_SIZE_KB'
| 'LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND'
| 'LOGS_LIMITER_TTL_SECONDS'
>
const DEFAULT_LIMITER_NAME = 'logs-rate-limiter'
/** Redis key prefix for a given limiter name. Each ingestion type (logs, traces) gets its own
* namespace so their token buckets don't share — and deplete — the same per-team state. */
export const buildBaseRedisKey = (limiterName: string = DEFAULT_LIMITER_NAME): string =>
process.env.NODE_ENV == 'test' ? `@posthog-test/${limiterName}` : `@posthog/${limiterName}`
/** Re-exported for the consumer test which needs to clean up keys between runs. */
export const BASE_REDIS_KEY = buildBaseRedisKey()
export type LogsRateLimit = {
tokensBefore: number
tokensAfter: number
isRateLimited: boolean
}
export type FilteredMessages = {
allowed: LogsIngestionMessage[]
dropped: LogsIngestionMessage[]
}
/**
* The LogsRateLimiterService is used to rate limit logs ingestion to ensure we aren't allowing too many logs to be ingested at once.
* The key part is we specify the refill rate as our per second KB/s limit. and the bucket size as the amount we are allowed to burst to.
* The burst shouldn't be too much higher.
*/
export class LogsRateLimiterService {
private teamBucketSizes: Map<number, number>
private teamRefillRates: Map<number, number>
private disabledTeamIds: Set<number> | '*' | null
private enabledTeamIds: Set<number> | '*' | null
private rateLimiter: KeyedRateLimiterService
constructor(
private config: LogsRateLimiterConfig,
redis: RedisV2,
limiterName: string = DEFAULT_LIMITER_NAME
) {
this.teamBucketSizes = this.parseTeamConfig(config.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB)
this.teamRefillRates = this.parseTeamConfig(config.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND)
this.disabledTeamIds = this.parseTeamIdList(config.LOGS_LIMITER_DISABLED_FOR_TEAMS)
this.enabledTeamIds = this.parseTeamIdList(config.LOGS_LIMITER_ENABLED_TEAMS)
this.rateLimiter = new KeyedRateLimiterService(
// Bucket params are passed per-request so live `this.config` mutations
// (used in tests) take effect. failOpen=false preserves the legacy
// throw-on-Redis-outage behaviour.
{ name: limiterName, failOpen: false },
redis
)
}
private parseTeamIdList(config: string): Set<number> | '*' | null {
if (config === '*') {
return '*'
}
if (!config) {
return null
}
const ids = new Set<number>()
for (const id of config.split(',')) {
const parsed = parseInt(id.trim(), 10)
if (!isNaN(parsed)) {
ids.add(parsed)
}
}
return ids
}
private parseTeamConfig(config: string): Map<number, number> {
const result = new Map<number, number>()
if (!config) {
return result
}
for (const entry of config.split(',')) {
const [teamId, value] = entry.split(':').map((s) => parseInt(s.trim(), 10))
if (!isNaN(teamId) && !isNaN(value)) {
result.set(teamId, value)
}
}
return result
}
private getHeaderValue(headers: any[] | undefined, key: string): string | undefined {
if (!headers || !Array.isArray(headers)) {
return undefined
}
for (const header of headers) {
if (header[key]) {
// Convert Buffer to string
return Buffer.from(header[key]).toString('utf8')
}
}
return undefined
}
private getTimestampFromMessage(message: LogsIngestionMessage): number {
const createdAtHeader = this.getHeaderValue(message.message.headers, 'created_at')
if (createdAtHeader) {
const timestamp = Date.parse(createdAtHeader) // Parse RFC3339/ISO8601 string
if (!isNaN(timestamp)) {
return msToSeconds(timestamp)
}
}
// Fallback to current time
return msToSeconds(Date.now())
}
public async rateLimitMany(idCosts: [string, number, number][]): Promise<[string, LogsRateLimit][]> {
const requests: KeyedRateLimitRequest[] = idCosts.map(([id, cost, nowSeconds]) => {
const teamId = parseInt(id, 10)
return {
id,
cost,
now: nowSeconds,
bucketSize: this.teamBucketSizes.get(teamId) ?? this.config.LOGS_LIMITER_BUCKET_SIZE_KB,
refillRate: this.teamRefillRates.get(teamId) ?? this.config.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND,
ttlSeconds: this.config.LOGS_LIMITER_TTL_SECONDS,
}
})
const results = await this.rateLimiter.rateLimitMany(requests)
return results.map(([id, r]) => [
id,
{ tokensBefore: r.tokensBefore, tokensAfter: r.tokens, isRateLimited: r.isRateLimited },
])
}
private isRateLimitingEnabledForTeam(teamId: number): boolean {
if (this.disabledTeamIds === '*') {
return false
}
if (this.disabledTeamIds?.has(teamId)) {
return false
}
if (this.enabledTeamIds === '*') {
return true
}
if (!this.enabledTeamIds) {
return false
}
return this.enabledTeamIds.has(teamId)
}
public async filterMessages(messages: LogsIngestionMessage[]): Promise<FilteredMessages> {
// Group messages by team to calculate total cost per team (only for teams with rate limiting enabled)
const teamCosts = new Map<number, number>()
const teamOldestTimestamps = new Map<number, number>()
const teamNewestTimestamps = new Map<number, number>()
for (const message of messages) {
const messageTimestamp = this.getTimestampFromMessage(message)
const existingOldest = teamOldestTimestamps.get(message.teamId)
if (existingOldest === undefined || messageTimestamp < existingOldest) {
teamOldestTimestamps.set(message.teamId, messageTimestamp)
}
const existingNewest = teamNewestTimestamps.get(message.teamId)
if (existingNewest === undefined || messageTimestamp > existingNewest) {
teamNewestTimestamps.set(message.teamId, messageTimestamp)
}
if (!this.isRateLimitingEnabledForTeam(message.teamId)) {
continue
}
const currentCost = teamCosts.get(message.teamId) ?? 0
// Cost is in KB (uncompressed bytes / 1000)
const costKb = bytesToKb(message.bytesUncompressed)
teamCosts.set(message.teamId, currentCost + costKb)
}
// Track how far behind wall clock the messages are (using oldest per team for worst-case lag)
const nowSeconds = msToSeconds(Date.now())
for (const [, timestamp] of teamOldestTimestamps) {
logsMessageLagHistogram.observe(nowSeconds - timestamp)
}
// Use the newest timestamp per team for the token bucket: this gives the bucket credit
// for the full producer-time span the batch covers, so a catch-up batch refills against
// its own time range instead of seeing timeDiff=0 from the previous batch's last call.
const rateLimitResults = await this.rateLimitMany(
Array.from(teamCosts.entries()).map(([teamId, cost]) => [
teamId.toString(),
cost,
teamNewestTimestamps.get(teamId) ?? msToSeconds(Date.now()),
])
)
// Build a map of team rate limit results
const teamLimits = new Map<number, { tokensBefore: number; tokensAfter: number; isRateLimited: boolean }>()
for (const [teamIdStr, result] of rateLimitResults) {
teamLimits.set(parseInt(teamIdStr, 10), result)
}
// Filter messages based on rate limits, allowing partial batches through
const allowed: LogsIngestionMessage[] = []
const dropped: LogsIngestionMessage[] = []
const teamKbUsed = new Map<number, number>()
for (const message of messages) {
const limit = teamLimits.get(message.teamId)
if (!limit) {
// No rate limit for this team (either not enabled or not in the map)
allowed.push(message)
continue
}
const kbUsed = teamKbUsed.get(message.teamId) ?? 0
const availableKb = limit.tokensBefore
const messageKb = bytesToKb(message.bytesUncompressed)
// Allow message if we haven't exceeded the available tokens
if (kbUsed + messageKb <= availableKb) {
allowed.push(message)
teamKbUsed.set(message.teamId, kbUsed + messageKb)
} else {
dropped.push(message)
}
}
return { allowed, dropped }
}
}