Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

30.8 KB
Raw Blame
import { randomUUID } from 'crypto'

import { deleteKeysWithPrefix } from '~/common/redis/_tests/redis'
import { RedisV2, createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { logger } from '~/common/utils/logger'

import { Hub, ProjectId, Team } from '../../../types'
import { createExampleInvocation, createHogFunction } from '../../_tests/fixtures'
import { CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult, HogFunctionType } from '../../types'
import { createInvocationResult } from '../../utils/invocation-utils'
import { BASE_REDIS_KEY, HogWatcherConfig, HogWatcherService, HogWatcherState } from './hog-watcher.service'

jest.mock('~/common/utils/posthog', () => ({ captureTeamEvent: jest.fn() }))

const mockNow: jest.SpyInstance = jest.spyOn(Date, 'now')
const mockCaptureTeamEvent: jest.Mock = require('~/common/utils/posthog').captureTeamEvent as any

const DEFAULT_WATCHER_CONFIG: HogWatcherConfig = {
    hogCostTimingLowerMs: 50,
    hogCostTimingUpperMs: 550,
    hogCostTiming: 100,
    asyncCostTimingLowerMs: 100,
    asyncCostTimingUpperMs: 5000,
    asyncCostTiming: 20,
    sendEvents: true,
    bucketSize: 10000,
    refillRate: 10,
    ttl: 86400,
    automaticallyDisableFunctions: true,
    thresholdDegraded: 0.8,
    stateLockTtl: 60,
    observeResultsBufferTimeMs: 500,
    observeResultsBufferMaxResults: 500,
}

describe('HogWatcher', () => {
    let now: number
    let hub: Hub
    let watcher: HogWatcherService
    let watcherConfig: HogWatcherConfig
    let onStateChangeSpy: jest.SpyInstance
    let redis: RedisV2
    const hogFunctionId = randomUUID()
    let hogFunction: HogFunctionType

    let team: Team

    beforeAll(async () => {
        team = {
            id: 2,
            project_id: 1 as ProjectId,
            uuid: 'test-uuid',
            organization_id: 'organization-id',
            name: 'testTeam',
        } as Team
        hub = await createHub()
        jest.spyOn(hub.teamManager, 'getTeam').mockResolvedValue(team)
        redis = createRedisV2PoolFromConfig({
            connection: hub.CDP_REDIS_HOST
                ? { url: hub.CDP_REDIS_HOST, options: { port: hub.CDP_REDIS_PORT, password: hub.CDP_REDIS_PASSWORD } }
                : { url: hub.REDIS_URL },
            poolMinSize: hub.REDIS_POOL_MIN_SIZE,
            poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
        })
        process.env.CDP_HOG_WATCHER_2_ENABLED = 'true'
        process.env.CDP_HOG_WATCHER_2_CAPTURE_ENABLED = 'true'
    })

    beforeEach(async () => {
        now = 1720000000000
        mockNow.mockReturnValue(now)
        await Promise.all([
            deleteKeysWithPrefix(redis, `${BASE_REDIS_KEY}/tokens/${hogFunctionId}`),
            deleteKeysWithPrefix(redis, `${BASE_REDIS_KEY}/state/${hogFunctionId}`),
            deleteKeysWithPrefix(redis, `${BASE_REDIS_KEY}/state-lock/${hogFunctionId}`),
        ])
        watcherConfig = { ...DEFAULT_WATCHER_CONFIG }

        watcher = new HogWatcherService(hub.teamManager, watcherConfig, redis)
        onStateChangeSpy = jest.spyOn(watcher as any, 'onStateChange') as jest.SpyInstance
        hogFunction = createHogFunction({ id: hogFunctionId, team_id: 2 })
    })

    afterAll(async () => {
        await closeHub(hub)
    })

    const createResult = (options: {
        id?: string
        duration?: number
        finished?: boolean
        error?: string
        kind?: 'hog' | 'async_function'
    }): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> => {
        const invocation = createExampleInvocation({ id: options.id ?? hogFunctionId, team_id: 2 })
        invocation.state.timings = [
            {
                kind: options.kind ?? 'hog',
                duration_ms: options.duration ?? 0,
            },
        ]

        return createInvocationResult(
            invocation,
            {},
            {
                finished: options.finished ?? true,
                error: options.error,
            }
        )
    }

    const advanceTime = (ms: number) => {
        now += ms
        mockNow.mockReturnValue(now)
    }

    describe('constructor', () => {
        it('should validate the bounds configuration', () => {
            expect(() => {
                const _badWatcher = new HogWatcherService(
                    hub.teamManager,
                    {
                        ...DEFAULT_WATCHER_CONFIG,
                        hogCostTimingLowerMs: 100,
                        hogCostTimingUpperMs: 100,
                        hogCostTiming: 1,
                        asyncCostTimingLowerMs: 100,
                        asyncCostTimingUpperMs: 100,
                        asyncCostTiming: 1,
                    },
                    redis
                )
            }).toThrow(
                'Lower bound for kind hog of 100ms must be lower than upper bound of 100ms. This is a configuration error.'
            )
        })
    })

    describe('observeResults', () => {
        const cases: [
            { name: string; cost: number; state: number },
            CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>[],
        ][] = [
            [
                { name: 'should calculate cost and state for single default result', cost: 0, state: 1 },
                [createResult({})],
            ],
            [
                { name: 'should calculate cost and state for multiple default results', cost: 0, state: 1 },
                [createResult({}), createResult({}), createResult({})],
            ],
            [
                { name: 'should calculate cost and state for small durations', cost: 0, state: 1 },
                [createResult({ duration: 10 }), createResult({ duration: 20 }), createResult({ duration: 30 })],
            ],
            [
                { name: 'should calculate cost and state for medium durations', cost: 12, state: 1 },
                [
                    createResult({ duration: 1000, kind: 'async_function' }),
                    createResult({ duration: 1000, kind: 'async_function' }),
                    createResult({ duration: 1000, kind: 'async_function' }),
                ],
            ],
            [
                { name: 'should calculate cost and state for single large duration', cost: 20, state: 1 },
                [createResult({ duration: 5000, kind: 'async_function' })],
            ],
            [
                { name: 'should calculate cost and state for single very large duration', cost: 40, state: 1 },
                [createResult({ duration: 10000, kind: 'async_function' })],
            ],
            [
                {
                    name: 'should calculate cumulative cost and state for multiple large durations',
                    cost: 141,
                    state: 1,
                },
                [
                    createResult({ duration: 5000, kind: 'async_function' }),
                    createResult({ duration: 10000, kind: 'async_function' }),
                    createResult({ duration: 20000, kind: 'async_function' }),
                ],
            ],
        ]

        it.each(cases.map(([meta, results]) => [meta.name, meta, results]))(
            '%s',
            async (name, expectedScore, results) => {
                await watcher.observeResults(results)
                const result = await watcher.getPersistedState(hogFunctionId)
                expect(hub.CDP_WATCHER_BUCKET_SIZE - result.tokens).toEqual(expectedScore.cost)
                expect(result.state).toEqual(expectedScore.state)
            }
        )

        it('should calculate costs per individual timing not based on total duration', async () => {
            // Create a result with multiple timings that would have different costs
            // if calculated individually vs. summed together
            const result = createResult({
                id: 'id1',
                finished: true,
                kind: 'async_function',
            }) as CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction>

            // Replace the default timing with multiple timings
            result.invocation.state.timings = [
                { kind: 'async_function', duration_ms: 90 }, // Below threshold, should have minimal cost
                { kind: 'async_function', duration_ms: 90 }, // Below threshold, should have minimal cost
                { kind: 'async_function', duration_ms: 90 }, // Below threshold, should have minimal cost
            ]

            // If using individual timings (correct): each timing has a small cost
            // If using total duration (incorrect): 300ms total would have a higher cost

            await watcher.observeResults([result])
            const state = await watcher.getPersistedState(hogFunctionId)

            // Expected: each 100ms timing has minimal cost since it's below the lower threshold
            // This is checking that we're not summing them into a 300ms duration
            const expectedIndividualCost = 0 // Three 100ms timings each have minimal/zero cost
            const totalCost = hub.CDP_WATCHER_BUCKET_SIZE - state.tokens

            expect(totalCost).toEqual(expectedIndividualCost)
        })

        it('should max out scores', async () => {
            let lotsOfResults = Array(10000).fill(createResult({ duration: 25000, kind: 'async_function' }))

            await watcher.observeResults(lotsOfResults)

            // V3 lua floor-drains on overdraft, so the persisted pool lands at 0.
            // The state machine is driven by the rate-limiter return (tokens=-1 on the
            // denied per-input results inside observeResults), so `state: 3` (disabled).
            expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                {
                  "state": 3,
                  "tokens": 0,
                }
            `)

            lotsOfResults = Array(10000).fill(createResult({ id: 'id2', kind: 'async_function' }))

            await watcher.observeResults(lotsOfResults)

            expect(await watcher.getPersistedState('id2')).toMatchInlineSnapshot(`
                {
                  "state": 1,
                  "tokens": 10000,
                }
            `)
        })

        it('should refill over time', async () => {
            hub.CDP_WATCHER_REFILL_RATE = 10
            await watcher.observeResults([
                createResult({ duration: 10000, kind: 'async_function' }),
                createResult({ duration: 10000, kind: 'async_function' }),
                createResult({ duration: 10000, kind: 'async_function' }),
            ])

            expect((await watcher.getPersistedState(hogFunctionId)).tokens).toMatchInlineSnapshot(`9880`)
            advanceTime(1000)
            expect((await watcher.getPersistedState(hogFunctionId)).tokens).toMatchInlineSnapshot(`9890`)
            advanceTime(10000)
            expect((await watcher.getPersistedState(hogFunctionId)).tokens).toMatchInlineSnapshot(`9990`)
        })

        describe('onStateChange', () => {
            it('should trigger state change events', async () => {
                await watcher.clearLock(hogFunctionId) // For testing the logic
                await watcher.observeResults(Array(10).fill(createResult({ duration: 1000, kind: 'hog' })))
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 1,
                      "tokens": 8100,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(0)

                await watcher.clearLock(hogFunctionId) // For testing the logic
                await watcher.observeResults(Array(10).fill(createResult({ duration: 1000, kind: 'hog' })))
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 2,
                      "tokens": 6200,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(1) // New state change
                expect(onStateChangeSpy).toHaveBeenLastCalledWith({
                    hogFunction,
                    state: HogWatcherState.degraded,
                    previousState: HogWatcherState.healthy,
                })

                await watcher.clearLock(hogFunctionId) // For testing the logic
                await watcher.observeResults(Array(10).fill(createResult({ duration: 1000, kind: 'hog' })))
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 2,
                      "tokens": 4300,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(1) // NO New state change

                await watcher.clearLock(hogFunctionId) // For testing the logic
                await watcher.observeResults(Array(100).fill(createResult({ duration: 1000, kind: 'hog' })))
                // Final batch overdraws; V3 lua floor-drains the available tokens, so the
                // stored pool lands at 0. State transition fires from the rate limiter
                // return (tokens=-1 on the denied per-input tail).
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 3,
                      "tokens": 0,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(2) // New state change
                expect(onStateChangeSpy).toHaveBeenLastCalledWith({
                    hogFunction,
                    state: HogWatcherState.disabled,
                    previousState: HogWatcherState.degraded,
                })
            })

            it('should not transition to disabled if not enabled', async () => {
                watcherConfig.automaticallyDisableFunctions = false
                watcher = new HogWatcherService(hub.teamManager, watcherConfig, redis)
                onStateChangeSpy = jest.spyOn(watcher as any, 'onStateChange') as jest.SpyInstance
                await watcher.observeResults(Array(1000).fill(createResult({ duration: 1000, kind: 'hog' })))
                // Cold-start denial: V3 lua floor-drains to 0. State transition fires
                // from the rate-limiter return (tokens=-1 on denied per-input results).
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 2,
                      "tokens": 0,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(1)
                expect(onStateChangeSpy).toHaveBeenLastCalledWith({
                    hogFunction,
                    state: HogWatcherState.degraded,
                    previousState: HogWatcherState.healthy,
                })

                await watcher.observeResults(Array(1000).fill(createResult({ duration: 1000, kind: 'hog' })))
                expect(onStateChangeSpy).toHaveBeenCalledTimes(1)
            })

            it('should not automatically transition out of disabled', async () => {
                await watcher.observeResults(Array(1000).fill(createResult({ duration: 1000, kind: 'hog' })))
                // V3 lua floor-drains on overdraft → pool=0. State persists at 3
                // (disabled) because state writes are independent of pool writes.
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 3,
                      "tokens": 0,
                    }
                `)
                advanceTime(1000)
                // Refill from 0 at refillRate=10 over 1s.
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 3,
                      "tokens": 10,
                    }
                `)

                advanceTime(1000)
                await watcher.observeResults(Array(1).fill(createResult({ duration: 10, kind: 'hog' })))
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 3,
                      "tokens": 20,
                    }
                `)
                expect(onStateChangeSpy).toHaveBeenCalledTimes(1)
            })

            it('should not change states if recently changed', async () => {
                await watcher.doStageChanges([[hogFunction, HogWatcherState.healthy]])
                await watcher.observeResults(Array(1000).fill(createResult({ duration: 1000, kind: 'hog' })))
                expect((await watcher.getPersistedState(hogFunctionId)).state).toEqual(HogWatcherState.healthy)
                const res = await redis.usePipeline({ name: 'getLock' }, (pipeline) => {
                    pipeline.get(`@posthog-test/hog-watcher-2/state-lock/${hogFunctionId}`)
                    pipeline.ttl(`@posthog-test/hog-watcher-2/state-lock/${hogFunctionId}`)
                })
                expect(res?.[0]?.[1]).toEqual('1') // The value
                expect(res?.[1]?.[1]).toBeGreaterThan(hub.CDP_WATCHER_STATE_LOCK_TTL - 5) // The ttl
                expect(res?.[1]?.[1]).toBeLessThan(hub.CDP_WATCHER_STATE_LOCK_TTL + 5) // The ttl
            })

            it('should not transition to a different state if forcefully set', async () => {
                await watcher.doStageChanges([[hogFunction, HogWatcherState.forcefully_degraded]], true)
                await watcher.clearLock(hogFunctionId)
                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 11,
                      "tokens": 0,
                    }
                `)
                await watcher.observeResults(Array(1000).fill(createResult({ duration: 1, kind: 'hog' })))

                expect(await watcher.getPersistedState(hogFunctionId)).toMatchInlineSnapshot(`
                    {
                      "state": 11,
                      "tokens": 0,
                    }
                `)
            })
        })
    })

    describe('getPersistedStates', () => {
        it('should return healthy with full bucket for unknown function', async () => {
            const state = await watcher.getPersistedState('totally-unknown-function-id')
            expect(state).toEqual({
                state: HogWatcherState.healthy,
                tokens: 10000,
            })
        })

        it('should not create Redis keys on read-only access', async () => {
            const unknownId = 'never-executed-function'
            await watcher.getPersistedState(unknownId)
            await watcher.getPersistedState(unknownId)

            // Verify no key was created — reads should be side-effect-free
            const exists = await redis.useClient({ name: 'test-check' }, async (client) => {
                return await client.exists(`${BASE_REDIS_KEY}/tokens/${unknownId}`)
            })
            expect(exists).toEqual(0)
        })

        it('should cap refill at bucketSize after large time gap', async () => {
            await watcher.observeResults([createResult({ duration: 10000, kind: 'async_function' })])
            expect((await watcher.getPersistedState(hogFunctionId)).tokens).toBeLessThan(10000)

            // Advance time by a very large amount — tokens should cap at bucketSize, not overflow
            advanceTime(1_000_000_000)
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(state.tokens).toEqual(watcherConfig.bucketSize)
        })

        it('should handle pool existing without ts in Redis', async () => {
            // Write pool directly without ts to simulate corrupted state
            await redis.useClient({ name: 'test-setup' }, async (client) => {
                await client.hset(`${BASE_REDIS_KEY}/tokens/${hogFunctionId}`, 'pool', '5000')
            })

            const state = await watcher.getPersistedState(hogFunctionId)
            // With no ts, timeDiff is 0, so no refill — tokens should be the raw pool value
            expect(state.tokens).toEqual(5000)
        })

        it('should handle ts existing without pool in Redis', async () => {
            // Write ts directly without pool to simulate corrupted state
            await redis.useClient({ name: 'test-setup' }, async (client) => {
                await client.hset(`${BASE_REDIS_KEY}/tokens/${hogFunctionId}`, 'ts', Math.round(now / 1000))
            })

            const state = await watcher.getPersistedState(hogFunctionId)
            // With no pool, should return bucketSize (healthy default)
            expect(state.tokens).toEqual(watcherConfig.bucketSize)
        })

        it('should return consistent results between write and read paths', async () => {
            // Write costs via observeResults (uses evalsha Lua script)
            await watcher.observeResults([
                createResult({ duration: 5000, kind: 'async_function' }),
                createResult({ duration: 5000, kind: 'async_function' }),
            ])

            // Read immediately via getPersistedState (uses hget)
            const state = await watcher.getPersistedState(hogFunctionId)

            // Tokens should be reduced but function still healthy
            expect(state.tokens).toBeLessThan(watcherConfig.bucketSize)
            expect(state.tokens).toBeGreaterThan(0)
            expect(state.state).toEqual(HogWatcherState.healthy)
        })
    })

    describe('doStateChanges - with resetPool', () => {
        const expectMockCaptureTeamEvent = (state: string, previousState: string) => {
            expect(mockCaptureTeamEvent).toHaveBeenCalledWith(team, 'hog_function_state_change', {
                hog_function_id: hogFunction.id,
                hog_function_type: hogFunction.type,
                hog_function_name: hogFunction.name,
                hog_function_template_id: hogFunction.template_id,
                state,
                previous_state: previousState,
            })
        }

        it('should change the state of a hog function', async () => {
            expect(await watcher.getPersistedState(hogFunction.id)).toEqual({
                state: HogWatcherState.healthy,
                tokens: 10000,
            })
            await watcher.doStageChanges([[hogFunction, HogWatcherState.degraded]], true)
            expect(await watcher.getPersistedState(hogFunction.id)).toEqual({
                state: HogWatcherState.degraded,
                tokens: 8000,
            })

            expect(onStateChangeSpy).toHaveBeenCalledWith({
                hogFunction,
                state: HogWatcherState.degraded,
                previousState: HogWatcherState.healthy,
            })
        })

        it('should only trigger state change events if the state actually changed', async () => {
            await watcher.doStageChanges([[hogFunction, HogWatcherState.degraded]], true)
            expect(onStateChangeSpy).toHaveBeenCalledTimes(1)
            expect(onStateChangeSpy).toHaveBeenLastCalledWith({
                hogFunction,
                state: HogWatcherState.degraded,
                previousState: HogWatcherState.healthy,
            })
            expectMockCaptureTeamEvent('degraded', 'healthy')

            await watcher.doStageChanges([[hogFunction, HogWatcherState.degraded]], true)
            expect(onStateChangeSpy).toHaveBeenCalledTimes(1)
            await watcher.doStageChanges([[hogFunction, HogWatcherState.disabled]], true)
            expect(onStateChangeSpy).toHaveBeenCalledTimes(2)
            expect(onStateChangeSpy).toHaveBeenLastCalledWith({
                hogFunction,
                state: HogWatcherState.disabled,
                previousState: HogWatcherState.degraded,
            })
            expectMockCaptureTeamEvent('disabled', 'degraded')
            await watcher.doStageChanges([[hogFunction, HogWatcherState.disabled]], true)
            expect(onStateChangeSpy).toHaveBeenCalledTimes(2)
        })
    })

    describe('observeResultsBuffered', () => {
        let observeResultsSpy: jest.SpyInstance
        beforeEach(() => {
            observeResultsSpy = jest.spyOn(watcher, 'observeResults')
            watcherConfig.observeResultsBufferMaxResults = 3
        })

        it('should buffer results and observe them', async () => {
            const result1 = createResult({})
            const result2 = createResult({})
            const result3 = createResult({})

            await Promise.all([
                watcher.observeResultsBuffered(result1),
                watcher.observeResultsBuffered(result2),
                watcher.observeResultsBuffered(result3),
            ])
            expect(observeResultsSpy).toHaveBeenCalledTimes(1)
            expect(observeResultsSpy).toHaveBeenCalledWith([result1, result2, result3])
        })

        it('should buffer results and flush them when the buffer is full', async () => {
            const results = [createResult({}), createResult({}), createResult({}), createResult({})]
            await Promise.all([
                watcher.observeResultsBuffered(results[0]),
                watcher.observeResultsBuffered(results[1]),
                watcher.observeResultsBuffered(results[2]),
                watcher.observeResultsBuffered(results[3]),
            ])
            expect(observeResultsSpy).toHaveBeenCalledTimes(2)
            expect(observeResultsSpy).toHaveBeenCalledWith([results[0], results[1], results[2]])
            expect(observeResultsSpy).toHaveBeenCalledWith([results[3]])
        })
    })

    describe('reader fallback', () => {
        let loggerWarnSpy: jest.SpyInstance

        const makeFailingReader = (err: Error): RedisV2 => ({
            useClient: jest.fn(() => Promise.reject(err)),
            usePipeline: jest.fn(() => Promise.reject(err)),
        })

        beforeEach(() => {
            loggerWarnSpy = jest.spyOn(logger, 'warn').mockImplementation(() => undefined)
        })

        afterEach(() => {
            loggerWarnSpy.mockRestore()
        })

        it('should fall back to the writer when the reader throws on getPersistedStates', async () => {
            await watcher.observeResults([createResult({ duration: 1000, kind: 'hog' })])

            const failingReader = makeFailingReader(new Error('reader unavailable'))
            const watcherWithFailingReader = new HogWatcherService(hub.teamManager, watcherConfig, redis, failingReader)

            const state = await watcherWithFailingReader.getPersistedState(hogFunctionId)

            expect(failingReader.usePipeline).toHaveBeenCalledTimes(1)
            expect(state.tokens).toBeLessThan(watcherConfig.bucketSize)
            expect(loggerWarnSpy).toHaveBeenCalledWith(
                '🔀',
                expect.stringContaining('reader getStates failed'),
                expect.any(Object)
            )
        })

        it('should fall back to the writer when the reader throws on getAllFunctionStates', async () => {
            // Seed enough cost to trigger a state transition - that's what writes the
            // state key that getAllFunctionStates SCANs for.
            await watcher.observeResults(Array(10000).fill(createResult({ duration: 25000, kind: 'async_function' })))

            const failingReader = makeFailingReader(new Error('reader unavailable'))
            const watcherWithFailingReader = new HogWatcherService(hub.teamManager, watcherConfig, redis, failingReader)

            const states = await watcherWithFailingReader.getAllFunctionStates()

            expect(failingReader.useClient).toHaveBeenCalledTimes(1)
            expect(Object.keys(states)).toContain(hogFunctionId)
            expect(loggerWarnSpy).toHaveBeenCalledWith(
                '🔀',
                expect.stringContaining('reader scanStates failed'),
                expect.any(Object)
            )
        })

        it('should not log a fallback when the reader is healthy', async () => {
            const healthyWatcher = new HogWatcherService(hub.teamManager, watcherConfig, redis, redis)

            await healthyWatcher.getPersistedState(hogFunctionId)

            expect(loggerWarnSpy).not.toHaveBeenCalledWith(
                '🔀',
                expect.stringContaining('reader getStates failed'),
                expect.any(Object)
            )
        })
    })

    describe('observeAggregatedResults', () => {
        // Cost curve with the default config: lower=50ms, upper=550ms, cost=100.
        it('charges no cost when the aggregate duration is below the lower bound', async () => {
            await watcher.observeAggregatedResults([{ hogFunction, totalDurationMs: 50 }])
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(watcherConfig.bucketSize - state.tokens).toEqual(0)
        })

        it('charges the full cost at the upper bound', async () => {
            await watcher.observeAggregatedResults([{ hogFunction, totalDurationMs: 550 }])
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(watcherConfig.bucketSize - state.tokens).toEqual(100)
        })

        it('scales cost with total VM time past the upper bound', async () => {
            // (1050 - 50) / (550 - 50) = 2 → twice the max single-message cost
            await watcher.observeAggregatedResults([{ hogFunction, totalDurationMs: 1050 }])
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(watcherConfig.bucketSize - state.tokens).toEqual(200)
        })

        it('accumulates cost across observations for the same function', async () => {
            await watcher.observeAggregatedResults([
                { hogFunction, totalDurationMs: 300 },
                { hogFunction, totalDurationMs: 300 },
            ])
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(watcherConfig.bucketSize - state.tokens).toEqual(100) // 50 + 50
        })

        it('marks a function degraded once its bucket drops to the threshold', async () => {
            // cost 2000 leaves tokens at 8000 = 80% of the bucket → degraded
            await watcher.observeAggregatedResults([{ hogFunction, totalDurationMs: 10050 }])
            const state = await watcher.getPersistedState(hogFunctionId)
            expect(state.tokens).toEqual(8000)
            expect(state.state).toEqual(HogWatcherState.degraded)
        })

        it('charges each function independently', async () => {
            const other = createHogFunction({ id: `${hogFunctionId}-other`, team_id: 2 })
            await watcher.observeAggregatedResults([
                { hogFunction, totalDurationMs: 550 },
                { hogFunction: other, totalDurationMs: 50 },
            ])
            expect(watcherConfig.bucketSize - (await watcher.getPersistedState(hogFunctionId)).tokens).toEqual(100)
            expect(watcherConfig.bucketSize - (await watcher.getPersistedState('other-fn')).tokens).toEqual(0)
        })
    })
})