Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

37.4 KB
Raw Blame
import { deleteKeysWithPrefix } from '~/common/redis/_tests/redis'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { Hub } from '~/types'

import { BASE_REDIS_KEY, LogsRateLimiterService, logsMessageLagHistogram } from './logs-rate-limiter.service'

const mockNow: jest.SpyInstance = jest.spyOn(Date, 'now')

describe('LogsRateLimiterService', () => {
    jest.retryTimes(3, { logErrorsBeforeRetry: true })
    const LOGS_LIMITER_BUCKET_SIZE_KB = 100
    const BUCKET_EXCEEDED_UNCOMPRESSED_BYTES = 15360

    describe('integration', () => {
        let now: number
        let hub: Hub
        let rateLimiter: LogsRateLimiterService
        let redis: RedisV2
        const teamId1 = 'team-1'
        const teamId2 = 'team-2'

        beforeEach(async () => {
            hub = await createHub()
            now = 1720000000000
            mockNow.mockReturnValue(now)

            hub.LOGS_LIMITER_BUCKET_SIZE_KB = LOGS_LIMITER_BUCKET_SIZE_KB
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 10
            hub.LOGS_LIMITER_TTL_SECONDS = 60 * 60 * 24

            redis = createRedisV2PoolFromConfig({
                connection: hub.LOGS_REDIS_HOST
                    ? {
                          url: hub.LOGS_REDIS_HOST,
                          options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
                      }
                    : { url: hub.REDIS_URL },
                poolMinSize: hub.REDIS_POOL_MIN_SIZE,
                poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
            })
            await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)

            rateLimiter = new LogsRateLimiterService(hub, redis)
        })

        const advanceTime = (ms: number) => {
            now += ms
            mockNow.mockReturnValue(now)
        }

        afterEach(async () => {
            await closeHub(hub)
            jest.clearAllMocks()
        })

        it('should consume tokens and return before/after values', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 10, Math.round(now / 1000)]])

            expect(res).toEqual([[teamId1, { tokensBefore: 100, tokensAfter: 90, isRateLimited: false }]])
        })

        it('should rate limit when tokens are exhausted', async () => {
            let res = await rateLimiter.rateLimitMany([[teamId1, 99, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(100)
            expect(res[0][1].tokensAfter).toBe(1)
            expect(res[0][1].isRateLimited).toBe(false)

            res = await rateLimiter.rateLimitMany([[teamId1, 1, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(1)
            expect(res[0][1].tokensAfter).toBe(0)
            expect(res[0][1].isRateLimited).toBe(true)

            res = await rateLimiter.rateLimitMany([[teamId1, 20, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(0)
            expect(res[0][1].tokensAfter).toBe(-1)
            expect(res[0][1].isRateLimited).toBe(true)
        })

        it('should allow partial allowance calculation from tokensBefore', async () => {
            // First, consume 70 tokens, leaving 30
            await rateLimiter.rateLimitMany([[teamId1, 70, Math.round(now / 1000)]])

            // Request 50KB but only 30KB available - tokensBefore tells us how much to allow
            const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(30)
            expect(res[0][1].tokensAfter).toBe(-1)
            expect(res[0][1].isRateLimited).toBe(true)

            // Can calculate allowance ratio: 30KB available / 50KB requested = 60% allowance
            const allowedKb = res[0][1].tokensBefore
            const requestedKb = 50
            const allowanceRatio = allowedKb / requestedKb
            expect(allowanceRatio).toBeCloseTo(0.6)
        })

        it('should handle multiple teams independently', async () => {
            const res = await rateLimiter.rateLimitMany([
                [teamId1, 10, Math.round(now / 1000)],
                [teamId2, 50, Math.round(now / 1000)],
            ])

            expect(res).toEqual([
                [teamId1, { tokensBefore: 100, tokensAfter: 90, isRateLimited: false }],
                [teamId2, { tokensBefore: 100, tokensAfter: 50, isRateLimited: false }],
            ])

            const res2 = await rateLimiter.rateLimitMany([
                [teamId1, 5, Math.round(now / 1000)],
                [teamId2, 0, Math.round(now / 1000)],
            ])

            expect(res2).toEqual([
                [teamId1, { tokensBefore: 90, tokensAfter: 85, isRateLimited: false }],
                [teamId2, { tokensBefore: 50, tokensAfter: 50, isRateLimited: false }],
            ])
        })

        it('should refill tokens over time at configured rate', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(100)
            expect(res[0][1].tokensAfter).toBe(50)

            advanceTime(1000) // 1 second = 10KB refilled

            const res2 = await rateLimiter.rateLimitMany([[teamId1, 5, Math.round(now / 1000)]])

            expect(res2[0][1].tokensBefore).toBe(60) // 50 + 10 refilled
            expect(res2[0][1].tokensAfter).toBe(55)
            expect(res2[0][1].isRateLimited).toBe(false)

            advanceTime(4000) // 4 seconds = 40KB refilled

            const res3 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])

            expect(res3[0][1].tokensBefore).toBe(95) // 55 + 40, capped at 100
            expect(res3[0][1].tokensAfter).toBe(95)
            expect(res3[0][1].isRateLimited).toBe(false)
        })

        it('should expose accrued credit in tokensBefore but cap stored pool at bucket size', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 10, Math.round(now / 1000)]])

            expect(res[0][1].tokensAfter).toBe(90)

            advanceTime(5000) // 5 seconds at 10KB/s = 50KB of accrued refill credit

            const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])

            // tokensBefore reflects actual accrued credit (90 + 50 = 140) so that a single
            // catch-up call can spend more than the bucket size when the producer time span
            // earned that credit. The stored pool (tokensAfter, after deducting cost) is
            // still capped at poolMax to bound silent-period accumulation.
            expect(res2[0][1].tokensBefore).toBe(140)
            expect(res2[0][1].tokensAfter).toBe(100)
            expect(res2[0][1].isRateLimited).toBe(false)
        })

        it('should let a catch-up call redeem refill credit larger than the bucket', async () => {
            // Drain the bucket with a starting call.
            const start = await rateLimiter.rateLimitMany([[teamId1, 100, Math.round(now / 1000)]])
            expect(start[0][1].tokensAfter).toBe(0)

            // Simulate a 20s "lag" with no rate-limiter calls. With refill rate 10KB/s
            // that's 200KB of credit.
            advanceTime(20_000)

            // Single catch-up call costing the full accrued credit. Without uncapped
            // tokensBefore this would see tokensBefore=poolMax=100 and reject anything
            // beyond that — we should be able to consumer 200KB of credit without being rate-limited.
            const catchup = await rateLimiter.rateLimitMany([[teamId1, 199, Math.round(now / 1000)]])

            expect(catchup[0][1].tokensBefore).toBe(200) // = 0 prev + 20s × 10KB/s refill
            expect(catchup[0][1].tokensAfter).toBe(1)
            expect(catchup[0][1].isRateLimited).toBe(false)
        })

        it('should handle sequential requests for same team in single call', async () => {
            const res = await rateLimiter.rateLimitMany([
                [teamId1, 90, Math.round(now / 1000)],
                [teamId1, 9, Math.round(now / 1000)],
                [teamId1, 1, Math.round(now / 1000)],
                [teamId1, 2, Math.round(now / 1000)],
            ])

            expect(res).toEqual([
                [teamId1, { tokensBefore: 100, tokensAfter: 10, isRateLimited: false }],
                [teamId1, { tokensBefore: 10, tokensAfter: 1, isRateLimited: false }],
                [teamId1, { tokensBefore: 1, tokensAfter: 0, isRateLimited: true }],
                [teamId1, { tokensBefore: 0, tokensAfter: -1, isRateLimited: true }],
            ])
        })

        it('should handle zero cost requests', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])

            expect(res).toEqual([[teamId1, { tokensBefore: 100, tokensAfter: 100, isRateLimited: false }]])
        })

        it('should recover from empty pool over time', async () => {
            // Exhaust the bucket completely
            await rateLimiter.rateLimitMany([[teamId1, 100, Math.round(now / 1000)]])
            const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(0)
            expect(res[0][1].tokensAfter).toBe(-1)
            expect(res[0][1].isRateLimited).toBe(true)

            // Wait for refill (2 seconds = 20KB). The cost=50 denial above didn't
            // drain the bucket below 0 — denied requests don't deduct — so refill
            // starts from 0, not -1.
            advanceTime(2000)

            const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])

            expect(res2[0][1].tokensBefore).toBe(20)
            expect(res2[0][1].tokensAfter).toBe(20)
            expect(res2[0][1].isRateLimited).toBe(false)
        })

        it('should handle cost exceeding bucket size on first request', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 200, Math.round(now / 1000)]])

            expect(res[0][1].tokensBefore).toBe(100)
            expect(res[0][1].tokensAfter).toBe(-1)
            expect(res[0][1].isRateLimited).toBe(true)
        })

        it('should not refill within same second', async () => {
            const res = await rateLimiter.rateLimitMany([[teamId1, 50, Math.round(now / 1000)]])

            expect(res[0][1].tokensAfter).toBe(50)

            // Advance less than 1 second (use 400ms to avoid rounding to next second)
            advanceTime(400)

            const res2 = await rateLimiter.rateLimitMany([[teamId1, 0, Math.round(now / 1000)]])

            // No refill should occur
            expect(res2[0][1].tokensBefore).toBe(50)
            expect(res2[0][1].tokensAfter).toBe(50)
            expect(res2[0][1].isRateLimited).toBe(false)
        })
    })

    describe('filterMessages', () => {
        let hub: Hub
        let rateLimiter: LogsRateLimiterService
        let redis: RedisV2

        const createMessage = (teamId: number, bytesUncompressed: number): any => ({
            teamId,
            bytesUncompressed,
            bytesCompressed: Math.floor(bytesUncompressed / 2),
            recordCount: 1,
            token: `token-${teamId}`,
            message: { value: Buffer.from('test') },
        })

        beforeEach(async () => {
            hub = await createHub()
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = 10
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = 1
            hub.LOGS_LIMITER_TTL_SECONDS = 3600
            hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
            hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = ''
            hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = ''
            hub.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND = ''

            redis = createRedisV2PoolFromConfig({
                connection: hub.LOGS_REDIS_HOST
                    ? {
                          url: hub.LOGS_REDIS_HOST,
                          options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
                      }
                    : { url: hub.REDIS_URL },
                poolMinSize: hub.REDIS_POOL_MIN_SIZE,
                poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
            })
            await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)

            rateLimiter = new LogsRateLimiterService(hub, redis)
        })

        afterEach(async () => {
            await closeHub(hub)
        })

        it('should allow messages within bucket size', async () => {
            const messages = [createMessage(1, 5120), createMessage(1, 3072)] // 5KB + 3KB = 8KB < 10KB

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(2)
            expect(result.dropped).toHaveLength(0)
        })

        it('should drop messages exceeding bucket size', async () => {
            const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)] // 15KB > 10KB

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(0)
            expect(result.dropped).toHaveLength(1)
        })

        it('should allow partial batch through', async () => {
            const messages = [
                createMessage(1, 5120), // 5KB - allowed
                createMessage(1, 6144), // 6KB - dropped (5+6=11 > 10)
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.allowed[0].bytesUncompressed).toBe(5120)
            expect(result.dropped).toHaveLength(1)
            expect(result.dropped[0].bytesUncompressed).toBe(6144)
        })

        it('should rate limit teams independently', async () => {
            const messages = [
                createMessage(1, 15360), // Team 1: 15KB - dropped
                createMessage(2, 5120), // Team 2: 5KB - allowed
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.allowed[0].teamId).toBe(2)
            expect(result.dropped).toHaveLength(1)
            expect(result.dropped[0].teamId).toBe(1)
        })

        it('should skip rate limiting for teams not in LOGS_LIMITER_ENABLED_TEAMS', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '2' // Only team 2
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [
                createMessage(1, 15360), // Team 1: 15KB - allowed (not rate limited)
                createMessage(2, 15360), // Team 2: 15KB - dropped (rate limited)
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.allowed[0].teamId).toBe(1)
            expect(result.dropped).toHaveLength(1)
            expect(result.dropped[0].teamId).toBe(2)
        })

        it('should skip rate limiting when LOGS_LIMITER_ENABLED_TEAMS is empty', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = ''
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.dropped).toHaveLength(0)
        })

        it('should rate limit all teams when LOGS_LIMITER_ENABLED_TEAMS is *', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '*'

            const messages = [createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES)]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(0)
            expect(result.dropped).toHaveLength(1)
        })

        it('should handle multiple teams in LOGS_LIMITER_ENABLED_TEAMS', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '1, 3' // Teams 1 and 3
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [
                createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 1: dropped (rate limited)
                createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 2: allowed (not in list)
                createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES), // Team 3: dropped (rate limited)
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.allowed[0].teamId).toBe(2)
            expect(result.dropped).toHaveLength(2)
        })

        it('should skip rate limiting for all teams when LOGS_LIMITER_DISABLED_FOR_TEAMS is *', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
            hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '*'
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [
                createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
                createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
                createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(3)
            expect(result.dropped).toHaveLength(0)
        })

        it('should skip rate limiting for specific teams in LOGS_LIMITER_DISABLED_FOR_TEAMS', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '*'
            hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '1, 3'
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [
                createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
                createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
                createMessage(3, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(2)
            expect(result.allowed.map((m) => m.teamId)).toEqual([1, 3])
            expect(result.dropped).toHaveLength(1)
            expect(result.dropped[0].teamId).toBe(2)
        })

        it('should prioritize LOGS_LIMITER_DISABLED_FOR_TEAMS over LOGS_LIMITER_ENABLED_TEAMS', async () => {
            hub.LOGS_LIMITER_ENABLED_TEAMS = '1, 2'
            hub.LOGS_LIMITER_DISABLED_FOR_TEAMS = '1'
            rateLimiter = new LogsRateLimiterService(hub, redis)

            const messages = [
                createMessage(1, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
                createMessage(2, BUCKET_EXCEEDED_UNCOMPRESSED_BYTES),
            ]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.allowed[0].teamId).toBe(1)
            expect(result.dropped).toHaveLength(1)
            expect(result.dropped[0].teamId).toBe(2)
        })

        it('should handle zero byte messages', async () => {
            const messages = [createMessage(1, 0)]

            const result = await rateLimiter.filterMessages(messages)

            expect(result.allowed).toHaveLength(1)
            expect(result.dropped).toHaveLength(0)
        })

        describe('created_at header parsing', () => {
            const createMessageWithHeaders = (
                teamId: number,
                bytesUncompressed: number,
                headers?: Record<string, string>[]
            ): any => ({
                teamId,
                bytesUncompressed,
                bytesCompressed: Math.floor(bytesUncompressed / 2),
                recordCount: 1,
                token: `token-${teamId}`,
                message: {
                    value: Buffer.from('test'),
                    headers: headers?.map((h) => {
                        const result: Record<string, Buffer> = {}
                        for (const [key, value] of Object.entries(h)) {
                            result[key] = Buffer.from(value)
                        }
                        return result
                    }),
                },
            })

            it('should use created_at header timestamp for rate limiting', async () => {
                // First message at time T uses most of the bucket
                const pastTime = new Date('2024-01-01T00:00:00Z').toISOString()
                const messages1 = [createMessageWithHeaders(1, 8192, [{ created_at: pastTime }])] // 8KB

                const result1 = await rateLimiter.filterMessages(messages1)
                expect(result1.allowed).toHaveLength(1)
                expect(result1.dropped).toHaveLength(0)

                // Second message at same time T should be limited (only 2KB left)
                const messages2 = [createMessageWithHeaders(1, 5120, [{ created_at: pastTime }])] // 5KB

                const result2 = await rateLimiter.filterMessages(messages2)
                expect(result2.allowed).toHaveLength(0)
                expect(result2.dropped).toHaveLength(1)

                // Third message 10 seconds later should have refilled (10KB refilled at 1KB/s)
                const laterTime = new Date('2024-01-01T00:00:10Z').toISOString()
                const messages3 = [createMessageWithHeaders(1, 5120, [{ created_at: laterTime }])] // 5KB

                const result3 = await rateLimiter.filterMessages(messages3)
                expect(result3.allowed).toHaveLength(1)
                expect(result3.dropped).toHaveLength(0)
            })

            it('should fallback to current time when created_at header is missing', async () => {
                const messages = [createMessageWithHeaders(1, 5120)] // No headers

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })

            it('should fallback to current time when created_at header is invalid', async () => {
                const messages = [createMessageWithHeaders(1, 5120, [{ created_at: 'invalid-date' }])]

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })

            it('should handle RFC3339 date format', async () => {
                const rfcTime = '2024-06-15T14:30:00.000Z'
                const messages = [createMessageWithHeaders(1, 5120, [{ created_at: rfcTime }])]

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })

            it('should handle ISO8601 date format with timezone offset', async () => {
                const isoTime = '2024-06-15T14:30:00+02:00'
                const messages = [createMessageWithHeaders(1, 5120, [{ created_at: isoTime }])]

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })

            it('should observe lag histogram for teams without rate limiting enabled', async () => {
                hub.LOGS_LIMITER_ENABLED_TEAMS = '999' // Only team 999 is rate-limited
                rateLimiter = new LogsRateLimiterService(hub, redis)

                const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')

                const pastTime = new Date('2024-01-01T00:00:00Z').toISOString()
                const nowMs = new Date('2024-01-01T00:00:45Z').getTime()
                mockNow.mockReturnValue(nowMs)

                const messages = [createMessageWithHeaders(1, 1024, [{ created_at: pastTime }])]

                await rateLimiter.filterMessages(messages)

                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(45)

                observeSpy.mockRestore()
            })

            it('should observe lag histogram with Date.now() fallback when no timestamp header', async () => {
                const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')

                const nowMs = new Date('2024-01-01T00:01:00Z').getTime()
                mockNow.mockReturnValue(nowMs)

                const messages = [createMessageWithHeaders(1, 1024)] // no headers

                await rateLimiter.filterMessages(messages)

                // Lag should be ~0 since both message timestamp and nowSeconds use Date.now()
                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(0)

                observeSpy.mockRestore()
            })

            it('should observe lag histogram using oldest timestamp per team', async () => {
                const observeSpy = jest.spyOn(logsMessageLagHistogram, 'observe')

                const olderTime = new Date('2024-01-01T00:00:00Z').toISOString() // oldest
                const newerTime = new Date('2024-01-01T00:00:30Z').toISOString() // 30s later

                const nowMs = new Date('2024-01-01T00:01:00Z').getTime() // 60s after older, 30s after newer
                mockNow.mockReturnValue(nowMs)

                const messages = [
                    createMessageWithHeaders(1, 1024, [{ created_at: newerTime }]),
                    createMessageWithHeaders(1, 1024, [{ created_at: olderTime }]),
                ]

                await rateLimiter.filterMessages(messages)

                // Should observe lag of 60s (using oldest timestamp), not 30s (first message)
                expect(observeSpy).toHaveBeenCalledTimes(1)
                expect(observeSpy).toHaveBeenCalledWith(60)

                observeSpy.mockRestore()
            })

            it('should use max message timestamp for team when multiple messages in batch', async () => {
                const time1 = new Date('2024-01-01T00:00:00Z').toISOString()
                const time2 = new Date('2024-01-01T00:00:10Z').toISOString() // 10 seconds later

                // Both messages for same team, max timestamp should be used for refill calculation
                const messages = [
                    createMessageWithHeaders(1, 5120, [{ created_at: time1 }]), // 5KB
                    createMessageWithHeaders(1, 3072, [{ created_at: time2 }]), // 3KB
                ]

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(2)
                expect(result.dropped).toHaveLength(0)
            })

            it('should refill from newest timestamp when batch spans an old timestamp (catch-up scenario)', async () => {
                // First batch drains the bucket at t0.
                const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
                const messages1 = [createMessageWithHeaders(1, 10000, [{ created_at: t0 }])] // 10KB = full bucket
                const result1 = await rateLimiter.filterMessages(messages1)
                expect(result1.allowed).toHaveLength(1)

                // Second batch (catch-up) spans 10 seconds — oldest message is at t0 (matches previous
                // batch), newest at t10. Refill rate is 1KB/s, so 10 seconds of producer time => 10KB.
                // If we used the oldest timestamp the Lua would compute timeDiff=0 and not refill,
                // dropping the messages.
                const t10 = new Date('2024-01-01T00:00:10Z').toISOString()
                const messages2 = [
                    createMessageWithHeaders(1, 5120, [{ created_at: t0 }]), // 5KB at t0
                    createMessageWithHeaders(1, 3072, [{ created_at: t10 }]), // 3KB at t10
                ]

                const result2 = await rateLimiter.filterMessages(messages2)

                expect(result2.allowed).toHaveLength(2)
                expect(result2.dropped).toHaveLength(0)
            })

            it('should allow more throughput when batch spans time (backfill scenario)', async () => {
                // Simulate backfill: first batch drains the bucket
                const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
                const messages1 = [createMessageWithHeaders(1, 10000, [{ created_at: t0 }])] // 10KB = full bucket
                const result1 = await rateLimiter.filterMessages(messages1)
                expect(result1.allowed).toHaveLength(1)

                // Second batch arrives 10 seconds later.
                // Bucket is empty, refill rate is 1KB/s, so 10 seconds of refill = 10KB.
                // filterMessages uses the newest timestamp per team for refill, so the batch
                // gets credit for the full producer-time span it covers.
                const t10 = new Date('2024-01-01T00:00:10Z').toISOString()
                const messages2 = [
                    createMessageWithHeaders(1, 5120, [{ created_at: t10 }]), // 5KB at t0+10s
                    createMessageWithHeaders(1, 3072, [{ created_at: t10 }]), // 3KB at t0+10s
                ]

                const result2 = await rateLimiter.filterMessages(messages2)

                // Newest timestamp t10 gives 10KB refill from t0, enough for 5+3=8KB
                expect(result2.allowed).toHaveLength(2)
                expect(result2.dropped).toHaveLength(0)
            })

            it('should allow a catch-up batch larger than the bucket when refill credit covers it', async () => {
                // Simulate steady-state ingestion landing the bucket near full at t0.
                const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
                const result0 = await rateLimiter.filterMessages([
                    createMessageWithHeaders(1, 1000, [{ created_at: t0 }]),
                ])
                expect(result0.allowed).toHaveLength(1)

                // Now a single "catch-up" batch arrives covering 60 seconds of producer
                // time at the configured rate (1KB/s × 60s = 60KB), six times the bucket
                // size. Without uncapped tokensBefore, only ~10KB would pass.
                const messages: any[] = []
                for (let i = 1; i <= 60; i++) {
                    const ts = new Date(`2024-01-01T00:01:${(i - 1).toString().padStart(2, '0')}Z`).toISOString()
                    messages.push(createMessageWithHeaders(1, 1000, [{ created_at: ts }]))
                }

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(60)
                expect(result.dropped).toHaveLength(0)
            })

            it('should still drop excess when catch-up batch exceeds the configured rate', async () => {
                // Prior state at t0 so subsequent batch is not a "first call".
                const t0 = new Date('2024-01-01T00:00:00Z').toISOString()
                await rateLimiter.filterMessages([createMessageWithHeaders(1, 1000, [{ created_at: t0 }])])

                // Producer ran at 2KB/s for 60s → 120KB total spanning 60s. The
                // configured rate is 1KB/s so ~half should be dropped.
                const messages: any[] = []
                for (let i = 0; i < 120; i++) {
                    const seconds = 0 + Math.floor(i / 2)
                    const ts = new Date(
                        `2024-01-01T00:0${Math.floor(seconds / 60)}:${(seconds % 60).toString().padStart(2, '0')}Z`
                    ).toISOString()
                    messages.push(createMessageWithHeaders(1, 1000, [{ created_at: ts }]))
                }

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed.length).toBeGreaterThan(50)
                expect(result.allowed.length).toBeLessThan(80)
                expect(result.dropped.length).toBeGreaterThan(40)
            })

            it('should handle empty headers array', async () => {
                const message: any = {
                    teamId: 1,
                    bytesUncompressed: 5120,
                    bytesCompressed: 2560,
                    recordCount: 1,
                    token: 'token-1',
                    message: { value: Buffer.from('test'), headers: [] },
                }

                const result = await rateLimiter.filterMessages([message])

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })

            it('should handle headers without created_at key', async () => {
                const messages = [createMessageWithHeaders(1, 5120, [{ other_header: 'value' }])]

                const result = await rateLimiter.filterMessages(messages)

                expect(result.allowed).toHaveLength(1)
                expect(result.dropped).toHaveLength(0)
            })
        })
    })

    describe('team-specific limits', () => {
        let hub: Hub
        let redis: RedisV2

        const DEFAULT_BUCKET_SIZE_KB = 100
        const DEFAULT_REFILL_RATE_KB_PER_SECOND = 10

        beforeEach(async () => {
            hub = await createHub()
            hub.LOGS_LIMITER_BUCKET_SIZE_KB = DEFAULT_BUCKET_SIZE_KB
            hub.LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND = DEFAULT_REFILL_RATE_KB_PER_SECOND
            hub.LOGS_LIMITER_TTL_SECONDS = 3600
            hub.LOGS_LIMITER_ENABLED_TEAMS = '*'

            redis = createRedisV2PoolFromConfig({
                connection: hub.LOGS_REDIS_HOST
                    ? {
                          url: hub.LOGS_REDIS_HOST,
                          options: { port: hub.LOGS_REDIS_PORT, tls: hub.LOGS_REDIS_TLS ? {} : undefined },
                      }
                    : { url: hub.REDIS_URL },
                poolMinSize: hub.REDIS_POOL_MIN_SIZE,
                poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
            })
            await deleteKeysWithPrefix(redis, BASE_REDIS_KEY)
        })

        afterEach(async () => {
            await closeHub(hub)
        })

        it.each([
            { config: '1:50', teamId: '1', expectedBucket: 50, description: 'team with override' },
            {
                config: '1:50',
                teamId: '2',
                expectedBucket: DEFAULT_BUCKET_SIZE_KB,
                description: 'team without override',
            },
            { config: '1:25, 2:50, 3:75', teamId: '1', expectedBucket: 25, description: 'first of multiple overrides' },
            { config: '1:25, 2:50, 3:75', teamId: '3', expectedBucket: 75, description: 'last of multiple overrides' },
            { config: '', teamId: '1', expectedBucket: DEFAULT_BUCKET_SIZE_KB, description: 'empty config' },
        ])('uses correct bucket size for $description', async ({ config, teamId, expectedBucket }) => {
            hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = config
            const rateLimiter = new LogsRateLimiterService(hub, redis)

            const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])

            expect(result.tokensBefore).toBe(expectedBucket)
        })

        it.each([
            { config: '1:5', teamId: '1', expectedRefill: 10, description: 'team with 5KB/s override' },
            { config: '1:5', teamId: '2', expectedRefill: 20, description: 'team using default 10KB/s' },
        ])('uses correct refill rate for $description', async ({ config, teamId, expectedRefill }) => {
            hub.LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND = config
            const rateLimiter = new LogsRateLimiterService(hub, redis)

            await rateLimiter.rateLimitMany([[teamId, 50, Math.round(Date.now() / 1000)]])
            mockNow.mockReturnValue(Date.now() + 2000)
            const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])

            expect(result.tokensBefore).toBe(50 + expectedRefill)
        })

        it.each([
            { config: 'invalid', teamId: '1', description: 'completely invalid' },
            { config: '2:abc', teamId: '2', description: 'non-numeric value' },
            { config: ':100', teamId: '1', description: 'missing team id' },
            { config: '3:', teamId: '3', description: 'missing value' },
        ])('falls back to default for malformed config: $description', async ({ config, teamId }) => {
            hub.LOGS_LIMITER_TEAM_BUCKET_SIZE_KB = config
            const rateLimiter = new LogsRateLimiterService(hub, redis)

            const [[, result]] = await rateLimiter.rateLimitMany([[teamId, 0, Math.round(Date.now() / 1000)]])

            expect(result.tokensBefore).toBe(DEFAULT_BUCKET_SIZE_KB)
        })
    })

    // Construction does no Redis I/O, so these need neither a hub nor a live connection.
    describe('key namespacing', () => {
        const minimalConfig = {
            LOGS_LIMITER_TEAM_BUCKET_SIZE_KB: '',
            LOGS_LIMITER_TEAM_REFILL_RATE_KB_PER_SECOND: '',
            LOGS_LIMITER_DISABLED_FOR_TEAMS: '',
            LOGS_LIMITER_ENABLED_TEAMS: '*',
            LOGS_LIMITER_BUCKET_SIZE_KB: 100,
            LOGS_LIMITER_REFILL_RATE_KB_PER_SECOND: 10,
            LOGS_LIMITER_TTL_SECONDS: 3600,
        } as any

        it.each([
            {
                limiterName: undefined,
                expectedPrefix: '@posthog-test/logs-rate-limiter/tokens',
                description: 'defaults to the logs key prefix',
            },
            {
                limiterName: 'traces-rate-limiter',
                expectedPrefix: '@posthog-test/traces-rate-limiter/tokens',
                description: 'uses a distinct prefix when a limiter name is given (e.g. traces)',
            },
        ])('$description', ({ limiterName, expectedPrefix }) => {
            const rateLimiter = new LogsRateLimiterService(minimalConfig, {} as RedisV2, limiterName)
            expect(rateLimiter['rateLimiter'].getKeyPrefix()).toBe(expectedPrefix)
        })
    })
})