:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { deleteKeysWithPrefix } from '~/common/redis/_tests/redis'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { Hub } from '~/types'
import { BASE_REDIS_KEY, LogsRateLimiterService, logsMessageLagHistogram } from './logs-rate-limiter.service'
const mockNow: jest.SpyInstance = jest.spyOn(Date, 'now')
describe('LogsRateLimiterService', () => {
jest.retryTimes(3, { logErrorsBeforeRetry: true })
const LOGS_LIMITER_BUCKET_SIZE_KB = 100
const BUCKET_EXCEEDED_UNCOMPRESSED_BYTES = 15360
describe('integration', () => {
let now: number
let hub: Hub
let rateLimiter: LogsRateLimiterService
let redis: RedisV2
const teamId1 = 'team-1'
const teamId2 = 'team-2'
beforeEach(async () => {
hub = await createHub()
now = 1720000000000
mockNow.mockReturnValue(now)
hub.LOGS_LIMITER_BUCKET_SIZE_KB = LOGS_LIMITER_BUCKET_SIZE_KB
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 10
hub.LOGS_LIMITER_TTL_SECONDS = 60 * 60 * 24
redis = createRedisV2PoolFromConfig({
connection: hub.LOGS_REDIS_HOST
? {
url: hub.LOGS_REDIS_HOST,
options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)
rateLimiter = new LogsRateLimiterService(hub, redis)
})
const advanceTime = (ms: number) => {
now += ms
mockNow.mockReturnValue(now)
}
afterEach(async () => {
await closeHub(hub)
jest.clearAllMocks()
})
it('should consume tokens and return before/after values', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 10, Math.round(now / 1000)]])
expect(res).toEqual([[teamId1, { tokensBefore: 100, tokensAfter: 90, isRateLimited: false }]])
})
it('should rate limit when tokens are exhausted', async () => {
let res = await rateLimiter.rateLimitMany([[teamId1, 99, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(100)
expect(res[0][1].tokensAfter).toBe(1)
expect(res[0][1].isRateLimited).toBe(false)
res = await rateLimiter.rateLimitMany([[teamId1, 1, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(1)
expect(res[0][1].tokensAfter).toBe(0)
expect(res[0][1].isRateLimited).toBe(true)
res = await rateLimiter.rateLimitMany([[teamId1, 20, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(0)
expect(res[0][1].tokensAfter).toBe(-1)
expect(res[0][1].isRateLimited).toBe(true)
})
it('should allow partial allowance calculation from tokensBefore', async () => {
// First, consume 70 tokens, leaving 30
await rateLimiter.rateLimitMany([[teamId1, 70, Math.round(now / 1000)]])
// Request 50KB but only 30KB available - tokensBefore tells us how much to allow
const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(30)
expect(res[0][1].tokensAfter).toBe(-1)
expect(res[0][1].isRateLimited).toBe(true)
// Can calculate allowance ratio: 30KB available / 50KB requested = 60% allowance
const allowedKb = res[0][1].tokensBefore
const requestedKb = 50
const allowanceRatio = allowedKb / requestedKb
expect(allowanceRatio).toBeCloseTo(0.6)
})
it('should handle multiple teams independently', async () => {
const res = await rateLimiter.rateLimitMany([
[teamId1, 10, Math.round(now / 1000)],
[teamId2, 50, Math.round(now / 1000)],
])
expect(res).toEqual([
[teamId1, { tokensBefore: 100, tokensAfter: 90, isRateLimited: false }],
[teamId2, { tokensBefore: 100, tokensAfter: 50, isRateLimited: false }],
])
const res2 = await rateLimiter.rateLimitMany([
[teamId1, 5, Math.round(now / 1000)],
[teamId2, 0, Math.round(now / 1000)],
])
expect(res2).toEqual([
[teamId1, { tokensBefore: 90, tokensAfter: 85, isRateLimited: false }],
[teamId2, { tokensBefore: 50, tokensAfter: 50, isRateLimited: false }],
])
})
it('should refill tokens over time at configured rate', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(100)
expect(res[0][1].tokensAfter).toBe(50)
advanceTime(1000) // 1 second = 10KB refilled
const res2 = await rateLimiter.rateLimitMany([[teamId1, 5, Math.round(now / 1000)]])
expect(res2[0][1].tokensBefore).toBe(60) // 50 + 10 refilled
expect(res2[0][1].tokensAfter).toBe(55)
expect(res2[0][1].isRateLimited).toBe(false)
advanceTime(4000) // 4 seconds = 40KB refilled
const res3 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])
expect(res3[0][1].tokensBefore).toBe(95) // 55 + 40, capped at 100
expect(res3[0][1].tokensAfter).toBe(95)
expect(res3[0][1].isRateLimited).toBe(false)
})
it('should expose accrued credit in tokensBefore but cap stored pool at bucket size', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 10, Math.round(now / 1000)]])
expect(res[0][1].tokensAfter).toBe(90)
advanceTime(5000) // 5 seconds at 10KB/s = 50KB of accrued refill credit
const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])
// tokensBefore reflects actual accrued credit (90 + 50 = 140) so that a single
// catch-up call can spend more than the bucket size when the producer time span
// earned that credit. The stored pool (tokensAfter, after deducting cost) is
// still capped at poolMax to bound silent-period accumulation.
expect(res2[0][1].tokensBefore).toBe(140)
expect(res2[0][1].tokensAfter).toBe(100)
expect(res2[0][1].isRateLimited).toBe(false)
})
it('should let a catch-up call redeem refill credit larger than the bucket', async () => {
// Drain the bucket with a starting call.
const start = await rateLimiter.rateLimitMany([[teamId1, 100, Math.round(now / 1000)]])
expect(start[0][1].tokensAfter).toBe(0)
// Simulate a 20s "lag" with no rate-limiter calls. With refill rate 10KB/s
// that's 200KB of credit.
advanceTime(20_000)
// Single catch-up call costing the full accrued credit. Without uncapped
// tokensBefore this would see tokensBefore=poolMax=100 and reject anything
// beyond that — we should be able to consumer 200KB of credit without being rate-limited.
const catchup = await rateLimiter.rateLimitMany([[teamId1, 199, Math.round(now / 1000)]])
expect(catchup[0][1].tokensBefore).toBe(200) // = 0 prev + 20s × 10KB/s refill
expect(catchup[0][1].tokensAfter).toBe(1)
expect(catchup[0][1].isRateLimited).toBe(false)
})
it('should handle sequential requests for same team in single call', async () => {
const res = await rateLimiter.rateLimitMany([
[teamId1, 90, Math.round(now / 1000)],
[teamId1, 9, Math.round(now / 1000)],
[teamId1, 1, Math.round(now / 1000)],
[teamId1, 2, Math.round(now / 1000)],
])
expect(res).toEqual([
[teamId1, { tokensBefore: 100, tokensAfter: 10, isRateLimited: false }],
[teamId1, { tokensBefore: 10, tokensAfter: 1, isRateLimited: false }],
[teamId1, { tokensBefore: 1, tokensAfter: 0, isRateLimited: true }],
[teamId1, { tokensBefore: 0, tokensAfter: -1, isRateLimited: true }],
])
})
it('should handle zero cost requests', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])
expect(res).toEqual([[teamId1, { tokensBefore: 100, tokensAfter: 100, isRateLimited: false }]])
})
it('should recover from empty pool over time', async () => {
// Exhaust the bucket completely
await rateLimiter.rateLimitMany([[teamId1, 100, Math.round(now / 1000)]])
const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(0)
expect(res[0][1].tokensAfter).toBe(-1)
expect(res[0][1].isRateLimited).toBe(true)
// Wait for refill (2 seconds = 20KB). The cost=50 denial above didn't
// drain the bucket below 0 — denied requests don't deduct — so refill
// starts from 0, not -1.
advanceTime(2000)
const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])
expect(res2[0][1].tokensBefore).toBe(20)
expect(res2[0][1].tokensAfter).toBe(20)
expect(res2[0][1].isRateLimited).toBe(false)
})
it('should handle cost exceeding bucket size on first request', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 200, Math.round(now / 1000)]])
expect(res[0][1].tokensBefore).toBe(100)
expect(res[0][1].tokensAfter).toBe(-1)
expect(res[0][1].isRateLimited).toBe(true)
})
it('should not refill within same second', async () => {
const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])
expect(res[0][1].tokensAfter).toBe(50)
// Advance less than 1 second (use 400ms to avoid rounding to next second)
advanceTime(400)
const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])
// No refill should occur
expect(res2[0][1].tokensBefore).toBe(50)
expect(res2[0][1].tokensAfter).toBe(50)
expect(res2[0][1].isRateLimited).toBe(false)
})
})
describe('filterMessages', () => {
let hub: Hub
let rateLimiter: LogsRateLimiterService
let redis: RedisV2
const createMessage = (teamId: number, bytesUncompressed: number): any => ({
teamId,
bytesUncompressed,
bytesCompressed: Math.floor(bytesUncompressed / 2),
recordCount: 1,
token: `token-${teamId}`,
message: { value: Buffer.from('test') },
})
beforeEach(async () => {
hub = await createHub()
hub.LOGS_LIMITER_BUCKET_SIZE_KB = 10
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
hub.LOGS_LIMITER_TTL_SECONDS = 3600
hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = ''
hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = ''
hub.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND = ''
redis = createRedisV2PoolFromConfig({
connection: hub.LOGS_REDIS_HOST
? {
url: hub.LOGS_REDIS_HOST,
options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)
rateLimiter = new LogsRateLimiterService(hub, redis)
})
afterEach(async () => {
await closeHub(hub)
})
it('should allow messages within bucket size', async () => {
const messages = [createMessage(1, 5120), createMessage(1, 3072)] // 5KB + 3KB = 8KB < 10KB
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(2)
expect(result.dropped).toHaveLength(0)
})
it('should drop messages exceeding bucket size', async () => {
const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)] // 15KB > 10KB
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(0)
expect(result.dropped).toHaveLength(1)
})
it('should allow partial batch through', async () => {
const messages = [
createMessage(1, 5120), // 5KB - allowed
createMessage(1, 6144), // 6KB - dropped (5+6=11 > 10)
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.allowed[0].bytesUncompressed).toBe(5120)
expect(result.dropped).toHaveLength(1)
expect(result.dropped[0].bytesUncompressed).toBe(6144)
})
it('should rate limit teams independently', async () => {
const messages = [
createMessage(1, 15360), // Team 1: 15KB - dropped
createMessage(2, 5120), // Team 2: 5KB - allowed
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.allowed[0].teamId).toBe(2)
expect(result.dropped).toHaveLength(1)
expect(result.dropped[0].teamId).toBe(1)
})
it('should skip rate limiting for teams not in LOGS_LIMITER_ENABLED_TEAMS', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '2' // Only team 2
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [
createMessage(1, 15360), // Team 1: 15KB - allowed (not rate limited)
createMessage(2, 15360), // Team 2: 15KB - dropped (rate limited)
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.allowed[0].teamId).toBe(1)
expect(result.dropped).toHaveLength(1)
expect(result.dropped[0].teamId).toBe(2)
})
it('should skip rate limiting when LOGS_LIMITER_ENABLED_TEAMS is empty', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = ''
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should rate limit all teams when LOGS_LIMITER_ENABLED_TEAMS is *', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(0)
expect(result.dropped).toHaveLength(1)
})
it('should handle multiple teams in LOGS_LIMITER_ENABLED_TEAMS', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '1, 3' // Teams 1 and 3
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [
createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 1: dropped (rate limited)
createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 2: allowed (not in list)
createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 3: dropped (rate limited)
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.allowed[0].teamId).toBe(2)
expect(result.dropped).toHaveLength(2)
})
it('should skip rate limiting for all teams when LOGS_LIMITER_DISABLED_FOR_TEAMS is *', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '*'
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [
createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(3)
expect(result.dropped).toHaveLength(0)
})
it('should skip rate limiting for specific teams in LOGS_LIMITER_DISABLED_FOR_TEAMS', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '1, 3'
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [
createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(2)
expect(result.allowed.map((m) => m.teamId)).toEqual([1, 3])
expect(result.dropped).toHaveLength(1)
expect(result.dropped[0].teamId).toBe(2)
})
it('should prioritize LOGS_LIMITER_DISABLED_FOR_TEAMS over LOGS_LIMITER_ENABLED_TEAMS', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '1, 2'
hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '1'
rateLimiter = new LogsRateLimiterService(hub, redis)
const messages = [
createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.allowed[0].teamId).toBe(1)
expect(result.dropped).toHaveLength(1)
expect(result.dropped[0].teamId).toBe(2)
})
it('should handle zero byte messages', async () => {
const messages = [createMessage(1, 0)]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
describe('created_at header parsing', () => {
const createMessageWithHeaders = (
teamId: number,
bytesUncompressed: number,
headers?: Record<string, string>[]
): any => ({
teamId,
bytesUncompressed,
bytesCompressed: Math.floor(bytesUncompressed / 2),
recordCount: 1,
token: `token-${teamId}`,
message: {
value: Buffer.from('test'),
headers: headers?.map((h) => {
const result: Record<string, Buffer> = {}
for (const [key, value] of Object.entries(h)) {
result[key] = Buffer.from(value)
}
return result
}),
},
})
it('should use created_at header timestamp for rate limiting', async () => {
// First message at time T uses most of the bucket
const pastTime = new Date('2024-01-01T00:00:00Z').toISOString()
const messages1 = [createMessageWithHeaders(1, 8192, [{ created_at: pastTime }])] // 8KB
const result1 = await rateLimiter.filterMessages(messages1)
expect(result1.allowed).toHaveLength(1)
expect(result1.dropped).toHaveLength(0)
// Second message at same time T should be limited (only 2KB left)
const messages2 = [createMessageWithHeaders(1, 5120, [{ created_at: pastTime }])] // 5KB
const result2 = await rateLimiter.filterMessages(messages2)
expect(result2.allowed).toHaveLength(0)
expect(result2.dropped).toHaveLength(1)
// Third message 10 seconds later should have refilled (10KB refilled at 1KB/s)
const laterTime = new Date('2024-01-01T00:00:10Z').toISOString()
const messages3 = [createMessageWithHeaders(1, 5120, [{ created_at: laterTime }])] // 5KB
const result3 = await rateLimiter.filterMessages(messages3)
expect(result3.allowed).toHaveLength(1)
expect(result3.dropped).toHaveLength(0)
})
it('should fallback to current time when created_at header is missing', async () => {
const messages = [createMessageWithHeaders(1, 5120)] // No headers
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should fallback to current time when created_at header is invalid', async () => {
const messages = [createMessageWithHeaders(1, 5120, [{ created_at: 'invalid-date' }])]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should handle RFC3339 date format', async () => {
const rfcTime = '2024-06-15T14:30:00.000Z'
const messages = [createMessageWithHeaders(1, 5120, [{ created_at: rfcTime }])]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should handle ISO8601 date format with timezone offset', async () => {
const isoTime = '2024-06-15T14:30:00+02:00'
const messages = [createMessageWithHeaders(1, 5120, [{ created_at: isoTime }])]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should observe lag histogram for teams without rate limiting enabled', async () => {
hub.LOGS_LIMITER_ENABLED_TEAMS = '999' // Only team 999 is rate-limited
rateLimiter = new LogsRateLimiterService(hub, redis)
const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')
const pastTime = new Date('2024-01-01T00:00:00Z').toISOString()
const nowMs = new Date('2024-01-01T00:00:45Z').getTime()
mockNow.mockReturnValue(nowMs)
const messages = [createMessageWithHeaders(1, 1024, [{ created_at: pastTime }])]
await rateLimiter.filterMessages(messages)
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(45)
observeSpy.mockRestore()
})
it('should observe lag histogram with Date.now() fallback when no timestamp header', async () => {
const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')
const nowMs = new Date('2024-01-01T00:01:00Z').getTime()
mockNow.mockReturnValue(nowMs)
const messages = [createMessageWithHeaders(1, 1024)] // no headers
await rateLimiter.filterMessages(messages)
// Lag should be ~0 since both message timestamp and nowSeconds use Date.now()
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(0)
observeSpy.mockRestore()
})
it('should observe lag histogram using oldest timestamp per team', async () => {
const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')
const olderTime = new Date('2024-01-01T00:00:00Z').toISOString() // oldest
const newerTime = new Date('2024-01-01T00:00:30Z').toISOString() // 30s later
const nowMs = new Date('2024-01-01T00:01:00Z').getTime() // 60s after older, 30s after newer
mockNow.mockReturnValue(nowMs)
const messages = [
createMessageWithHeaders(1, 1024, [{ created_at: newerTime }]),
createMessageWithHeaders(1, 1024, [{ created_at: olderTime }]),
]
await rateLimiter.filterMessages(messages)
// Should observe lag of 60s (using oldest timestamp), not 30s (first message)
expect(observeSpy).toHaveBeenCalledTimes(1)
expect(observeSpy).toHaveBeenCalledWith(60)
observeSpy.mockRestore()
})
it('should use max message timestamp for team when multiple messages in batch', async () => {
const time1 = new Date('2024-01-01T00:00:00Z').toISOString()
const time2 = new Date('2024-01-01T00:00:10Z').toISOString() // 10 seconds later
// Both messages for same team, max timestamp should be used for refill calculation
const messages = [
createMessageWithHeaders(1, 5120, [{ created_at: time1 }]), // 5KB
createMessageWithHeaders(1, 3072, [{ created_at: time2 }]), // 3KB
]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(2)
expect(result.dropped).toHaveLength(0)
})
it('should refill from newest timestamp when batch spans an old timestamp (catch-up scenario)', async () => {
// First batch drains the bucket at t0.
const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
const messages1 = [createMessageWithHeaders(1, 10000, [{ created_at: t0 }])] // 10KB = full bucket
const result1 = await rateLimiter.filterMessages(messages1)
expect(result1.allowed).toHaveLength(1)
// Second batch (catch-up) spans 10 seconds — oldest message is at t0 (matches previous
// batch), newest at t10. Refill rate is 1KB/s, so 10 seconds of producer time => 10KB.
// If we used the oldest timestamp the Lua would compute timeDiff=0 and not refill,
// dropping the messages.
const t10 = new Date('2024-01-01T00:00:10Z').toISOString()
const messages2 = [
createMessageWithHeaders(1, 5120, [{ created_at: t0 }]), // 5KB at t0
createMessageWithHeaders(1, 3072, [{ created_at: t10 }]), // 3KB at t10
]
const result2 = await rateLimiter.filterMessages(messages2)
expect(result2.allowed).toHaveLength(2)
expect(result2.dropped).toHaveLength(0)
})
it('should allow more throughput when batch spans time (backfill scenario)', async () => {
// Simulate backfill: first batch drains the bucket
const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
const messages1 = [createMessageWithHeaders(1, 10000, [{ created_at: t0 }])] // 10KB = full bucket
const result1 = await rateLimiter.filterMessages(messages1)
expect(result1.allowed).toHaveLength(1)
// Second batch arrives 10 seconds later.
// Bucket is empty, refill rate is 1KB/s, so 10 seconds of refill = 10KB.
// filterMessages uses the newest timestamp per team for refill, so the batch
// gets credit for the full producer-time span it covers.
const t10 = new Date('2024-01-01T00:00:10Z').toISOString()
const messages2 = [
createMessageWithHeaders(1, 5120, [{ created_at: t10 }]), // 5KB at t0+10s
createMessageWithHeaders(1, 3072, [{ created_at: t10 }]), // 3KB at t0+10s
]
const result2 = await rateLimiter.filterMessages(messages2)
// Newest timestamp t10 gives 10KB refill from t0, enough for 5+3=8KB
expect(result2.allowed).toHaveLength(2)
expect(result2.dropped).toHaveLength(0)
})
it('should allow a catch-up batch larger than the bucket when refill credit covers it', async () => {
// Simulate steady-state ingestion landing the bucket near full at t0.
const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
const result0 = await rateLimiter.filterMessages([
createMessageWithHeaders(1, 1000, [{ created_at: t0 }]),
])
expect(result0.allowed).toHaveLength(1)
// Now a single "catch-up" batch arrives covering 60 seconds of producer
// time at the configured rate (1KB/s × 60s = 60KB), six times the bucket
// size. Without uncapped tokensBefore, only ~10KB would pass.
const messages: any[] = []
for (let i = 1; i <= 60; i++) {
const ts = new Date(`2024-01-01T00:01:${(i - 1).toString().padStart(2, '0')}Z`).toISOString()
messages.push(createMessageWithHeaders(1, 1000, [{ created_at: ts }]))
}
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(60)
expect(result.dropped).toHaveLength(0)
})
it('should still drop excess when catch-up batch exceeds the configured rate', async () => {
// Prior state at t0 so subsequent batch is not a "first call".
const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
await rateLimiter.filterMessages([createMessageWithHeaders(1, 1000, [{ created_at: t0 }])])
// Producer ran at 2KB/s for 60s → 120KB total spanning 60s. The
// configured rate is 1KB/s so ~half should be dropped.
const messages: any[] = []
for (let i = 0; i < 120; i++) {
const seconds = 0 + Math.floor(i / 2)
const ts = new Date(
`2024-01-01T00:0${Math.floor(seconds / 60)}:${(seconds % 60).toString().padStart(2, '0')}Z`
).toISOString()
messages.push(createMessageWithHeaders(1, 1000, [{ created_at: ts }]))
}
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed.length).toBeGreaterThan(50)
expect(result.allowed.length).toBeLessThan(80)
expect(result.dropped.length).toBeGreaterThan(40)
})
it('should handle empty headers array', async () => {
const message: any = {
teamId: 1,
bytesUncompressed: 5120,
bytesCompressed: 2560,
recordCount: 1,
token: 'token-1',
message: { value: Buffer.from('test'), headers: [] },
}
const result = await rateLimiter.filterMessages([message])
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
it('should handle headers without created_at key', async () => {
const messages = [createMessageWithHeaders(1, 5120, [{ other_header: 'value' }])]
const result = await rateLimiter.filterMessages(messages)
expect(result.allowed).toHaveLength(1)
expect(result.dropped).toHaveLength(0)
})
})
})
describe('team-specific limits', () => {
let hub: Hub
let redis: RedisV2
const DEFAULT_BUCKET_SIZE_KB = 100
const DEFAULT_REFILL_RATE_KB_PER_SECOND = 10
beforeEach(async () => {
hub = await createHub()
hub.LOGS_LIMITER_BUCKET_SIZE_KB = DEFAULT_BUCKET_SIZE_KB
hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = DEFAULT_REFILL_RATE_KB_PER_SECOND
hub.LOGS_LIMITER_TTL_SECONDS = 3600
hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
redis = createRedisV2PoolFromConfig({
connection: hub.LOGS_REDIS_HOST
? {
url: hub.LOGS_REDIS_HOST,
options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
}
: { url: hub.REDIS_URL },
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)
})
afterEach(async () => {
await closeHub(hub)
})
it.each([
{ config: '1:50', teamId: '1', expectedBucket: 50, description: 'team with override' },
{
config: '1:50',
teamId: '2',
expectedBucket: DEFAULT_BUCKET_SIZE_KB,
description: 'team without override',
},
{ config: '1:25, 2:50, 3:75', teamId: '1', expectedBucket: 25, description: 'first of multiple overrides' },
{ config: '1:25, 2:50, 3:75', teamId: '3', expectedBucket: 75, description: 'last of multiple overrides' },
{ config: '', teamId: '1', expectedBucket: DEFAULT_BUCKET_SIZE_KB, description: 'empty config' },
])('uses correct bucket size for $description', async ({ config, teamId, expectedBucket }) => {
hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = config
const rateLimiter = new LogsRateLimiterService(hub, redis)
const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])
expect(result.tokensBefore).toBe(expectedBucket)
})
it.each([
{ config: '1:5', teamId: '1', expectedRefill: 10, description: 'team with 5KB/s override' },
{ config: '1:5', teamId: '2', expectedRefill: 20, description: 'team using default 10KB/s' },
])('uses correct refill rate for $description', async ({ config, teamId, expectedRefill }) => {
hub.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND = config
const rateLimiter = new LogsRateLimiterService(hub, redis)
await rateLimiter.rateLimitMany([[teamId, 50, Math.round(Date.now() / 1000)]])
mockNow.mockReturnValue(Date.now() + 2000)
const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])
expect(result.tokensBefore).toBe(50 + expectedRefill)
})
it.each([
{ config: 'invalid', teamId: '1', description: 'completely invalid' },
{ config: '2:abc', teamId: '2', description: 'non-numeric value' },
{ config: ':100', teamId: '1', description: 'missing team id' },
{ config: '3:', teamId: '3', description: 'missing value' },
])('falls back to default for malformed config: $description', async ({ config, teamId }) => {
hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = config
const rateLimiter = new LogsRateLimiterService(hub, redis)
const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])
expect(result.tokensBefore).toBe(DEFAULT_BUCKET_SIZE_KB)
})
})
// Construction does no Redis I/O, so these need neither a hub nor a live connection.
describe('key namespacing', () => {
const minimalConfig = {
LOGS_LIMITER_TEAM_BUCKET_SIZE_KB: '',
LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND: '',
LOGS_LIMITER_DISABLED_FOR_TEAMS: '',
LOGS_LIMITER_ENABLED_TEAMS: '*',
LOGS_LIMITER_BUCKET_SIZE_KB: 100,
LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND: 10,
LOGS_LIMITER_TTL_SECONDS: 3600,
} as any
it.each([
{
limiterName: undefined,
expectedPrefix: '@posthog-test/logs-rate-limiter/tokens',
description: 'defaults to the logs key prefix',
},
{
limiterName: 'traces-rate-limiter',
expectedPrefix: '@posthog-test/traces-rate-limiter/tokens',
description: 'uses a distinct prefix when a limiter name is given (e.g. traces)',
},
])('$description', ({ limiterName, expectedPrefix }) => {
const rateLimiter = new LogsRateLimiterService(minimalConfig, {} as RedisV2, limiterName)
expect(rateLimiter['rateLimiter'].getKeyPrefix()).toBe(expectedPrefix)
})
})
})