Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

22.4 KB
Raw Blame
import { Counter } from 'prom-client'

import { RedisClientPipeline, RedisV2, getRedisPipelineResults } from '~/common/redis/redis-v2'
import { KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { LazyLoader } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { captureTeamEvent } from '~/common/utils/posthog'
import { TeamManager } from '~/common/utils/team-manager'

import {
    CyclotronJobInvocation,
    CyclotronJobInvocationHogFunction,
    CyclotronJobInvocationResult,
    HogFunctionTiming,
    HogFunctionType,
} from '../../types'

export interface HogWatcherConfig {
    hogCostTimingLowerMs: number
    hogCostTimingUpperMs: number
    hogCostTiming: number
    asyncCostTimingLowerMs: number
    asyncCostTimingUpperMs: number
    asyncCostTiming: number
    sendEvents: boolean
    bucketSize: number
    refillRate: number
    ttl: number
    automaticallyDisableFunctions: boolean
    thresholdDegraded: number
    stateLockTtl: number
    observeResultsBufferTimeMs: number
    observeResultsBufferMaxResults: number
}

export const BASE_REDIS_KEY = process.env.NODE_ENV == 'test' ? '@posthog-test/hog-watcher-2' : '@posthog/hog-watcher-2'
const REDIS_KEY_TOKENS = `${BASE_REDIS_KEY}/tokens`
const REDIS_KEY_STATE = `${BASE_REDIS_KEY}/state`
const REDIS_KEY_STATE_LOCK = `${BASE_REDIS_KEY}/state-lock`

export enum HogWatcherState {
    healthy = 1,
    degraded = 2,
    disabled = 3,

    // These are states that we do not auto transition into - can only be modified by the admin tool
    forcefully_degraded = 11,
    forcefully_disabled = 12,
}

export type HogWatcherFunctionState = {
    tokens: number
    state: HogWatcherState
}

/**
 * Compares a watcher read across Redis and Valkey on `state` alone, for `dualRead`.
 *
 * `tokens` is refilled from the wall clock at read time (see `getPersistedStates`), so the two
 * stores disagree on it whenever their reads land on different seconds. Comparing it would leave
 * the mismatch metric permanently saturated, and that metric is the signal we need to trust
 * before moving watcher reads to Valkey. `state` is what decides whether an invocation runs.
 */
export function sameWatcherState(
    primary: HogWatcherFunctionState | null,
    secondary: HogWatcherFunctionState | null
): boolean {
    return primary?.state === secondary?.state
}

/** `sameWatcherState` over a keyed batch — the id set has to agree as well as each state. */
export function sameWatcherStates(
    primary: Record<string, HogWatcherFunctionState>,
    secondary: Record<string, HogWatcherFunctionState>
): boolean {
    const ids = Object.keys(primary ?? {})
    return (
        ids.length === Object.keys(secondary ?? {}).length &&
        ids.every((id) => primary[id]?.state === secondary[id]?.state)
    )
}

type FunctionCostEntry = {
    hogFunction?: HogFunctionType
    functionId: string
    cost: number
}

const hogFunctionStateChange = new Counter({
    name: 'cdp_hog_function_state_change',
    help: 'Number of times a transformation state changed',
    labelNames: ['state', 'kind'],
})

type HogFunctionTimingCost = {
    lowerBound: number
    upperBound: number
    cost: number
}

type HogFunctionTimingCosts = Partial<Record<HogFunctionTiming['kind'], HogFunctionTimingCost>>

// Check if the result is of type CyclotronJobInvocationHogFunction
export const isHogFunctionResult = (
    result: CyclotronJobInvocationResult
): result is CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> => {
    return 'hogFunction' in result.invocation
}

// Helper if you don't care about the forced side of things
export const effectiveState = (state: HogWatcherState) => {
    if (state === HogWatcherState.forcefully_degraded) {
        return HogWatcherState.degraded
    }
    if (state === HogWatcherState.forcefully_disabled) {
        return HogWatcherState.disabled
    }
    return state
}

export class HogWatcherService {
    private costsMapping: HogFunctionTimingCosts
    private lazyLoader: LazyLoader<HogWatcherFunctionState>

    private queuedResults: {
        results: CyclotronJobInvocationResult[]
        promise: Promise<void>
        timeout: NodeJS.Timeout
        resolve: () => void
        reject: (error: unknown) => void
    } | null = null

    private redisReader: RedisV2
    private rateLimiter: KeyedRateLimiterService

    constructor(
        private teamManager: TeamManager,
        private config: HogWatcherConfig,
        private redis: RedisV2,
        redisReader?: RedisV2
    ) {
        this.redisReader = redisReader ?? redis
        // Token-bucket rate limiter — `name: 'hog-watcher-2'` produces the same
        // Redis key prefix this service has used historically (matches BASE_REDIS_KEY).
        this.rateLimiter = new KeyedRateLimiterService(
            {
                name: 'hog-watcher-2',
                bucketSize: this.config.bucketSize,
                refillRate: this.config.refillRate,
                ttlSeconds: this.config.ttl,
            },
            this.redis
        )
        this.costsMapping = {
            hog: {
                lowerBound: this.config.hogCostTimingLowerMs,
                upperBound: this.config.hogCostTimingUpperMs,
                cost: this.config.hogCostTiming,
            },
            async_function: {
                lowerBound: this.config.asyncCostTimingLowerMs,
                upperBound: this.config.asyncCostTimingUpperMs,
                cost: this.config.asyncCostTiming,
            },
        }

        for (const [kind, mapping] of Object.entries(this.costsMapping)) {
            if (mapping.lowerBound >= mapping.upperBound) {
                throw new Error(
                    `Lower bound for kind ${kind} of ${mapping.lowerBound}ms must be lower than upper bound of ${mapping.upperBound}ms. This is a configuration error.`
                )
            }
        }

        this.lazyLoader = new LazyLoader({
            name: 'hog_watcher_lazy_loader',
            refreshAgeMs: 30_000, // Cache for 30 seconds
            refreshJitterMs: 10_000,
            loader: async (ids) => await this.getPersistedStates(ids),
        })
    }

    private async onStateChange({
        hogFunction,
        state,
        previousState,
    }: {
        hogFunction: HogFunctionType
        state: HogWatcherState
        previousState: HogWatcherState
    }) {
        const team = await this.teamManager.getTeam(hogFunction.team_id)

        logger.info('[HogWatcherService] onStateChange', {
            hogFunctionId: hogFunction.id,
            hogFunctionName: hogFunction.name,
            state,
            previousState,
        })

        if (team && this.config.sendEvents) {
            captureTeamEvent(team, 'hog_function_state_change', {
                hog_function_id: hogFunction.id,
                hog_function_type: hogFunction.type,
                hog_function_name: hogFunction.name,
                hog_function_template_id: hogFunction.template_id,
                state: HogWatcherState[state], // Convert numeric state to readable string
                previous_state: HogWatcherState[previousState], // Convert numeric state to readable string
            })
        }
    }

    public calculateNewState(tokens: number): HogWatcherState {
        const rating = tokens / this.config.bucketSize

        if (rating < 0 && this.config.automaticallyDisableFunctions) {
            return HogWatcherState.disabled
        }
        if (rating <= this.config.thresholdDegraded) {
            return HogWatcherState.degraded
        }

        return HogWatcherState.healthy
    }

    /**
     * Get the persisted states of a list of hog functions.
     *
     * Uses plain hget calls instead of the evalsha Lua script.
     * The Lua script with cost=0 was doing 2 hget + 2 hset + 1 expire
     * per call — all writes unnecessary for a read-only check.
     * Replacing with 2 hget calls eliminates ~60% of internal Redis
     * operations per read and makes this method safe to route to
     * read replicas in the future.
     */
    public async getPersistedStates(
        ids: HogFunctionType['id'][]
    ): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
        const idsSet = new Set(ids)
        const nowSeconds = Math.round(Date.now() / 1000)

        const buildGetStatesPipeline = (pipeline: RedisClientPipeline) => {
            for (const id of idsSet) {
                pipeline.hmget(`${REDIS_KEY_TOKENS}/${id}`, 'pool', 'ts')
                pipeline.get(`${REDIS_KEY_STATE}/${id}`)
            }
        }

        let res
        try {
            res = await this.redisReader.usePipeline({ name: 'getStates' }, buildGetStatesPipeline)
        } catch (err) {
            logger.warn('🔀', '[HogWatcher] reader getStates failed, falling back to writer', { err })
            res = await this.redis.usePipeline({ name: 'getStates' }, buildGetStatesPipeline)
        }

        return Array.from(idsSet).reduce(
            (acc, id, index) => {
                const resIndex = index * 2
                const [pool, ts] = res?.[resIndex]?.[1] ?? [null, null]
                const stateVal = res?.[resIndex + 1]?.[1]

                // Same refill calculation as the Lua token bucket script
                let tokens: number
                if (pool === null || pool === undefined) {
                    tokens = this.config.bucketSize
                } else {
                    const timeDiff = ts ? Math.max(nowSeconds - Number(ts), 0) : 0
                    const owedTokens = timeDiff * this.config.refillRate
                    tokens = Math.min(Number(pool) + owedTokens, this.config.bucketSize)
                }

                acc[id] = {
                    state: stateVal ? Number(stateVal) : HogWatcherState.healthy,
                    tokens,
                }

                return acc
            },
            {} as Record<HogFunctionType['id'], HogWatcherFunctionState>
        )
    }

    /**
     * Like getPersistedStates but returns the state of a single hog function
     */
    public async getPersistedState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState> {
        const res = await this.getPersistedStates([id])
        return res[id]
    }

    public async getCachedPersistedState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState | null> {
        return await this.lazyLoader.get(id)
    }

    /**
     * Like getPersistedStates but returns the effective state (i.e. ignores forcefully set states)
     */
    public async getEffectiveStates(
        ids: HogFunctionType['id'][]
    ): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
        const states = await this.getPersistedStates(ids)
        return Object.fromEntries(
            Object.entries(states).map(([id, state]) => [
                id,
                { state: effectiveState(state.state), tokens: state.tokens },
            ])
        )
    }

    /**
     * Like getPersistedState but returns the effective state (i.e. ignores forcefully set states)
     */
    public async getEffectiveState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState> {
        const res = await this.getEffectiveStates([id])
        return res[id]
    }

    public async getCachedEffectiveState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState | null> {
        const res = await this.lazyLoader.get(id)
        if (!res) {
            return null
        }

        return { state: effectiveState(res.state), tokens: res.tokens }
    }

    public async getAllFunctionStates(): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
        const scan = (pool: RedisV2): Promise<string[] | null> =>
            pool.useClient({ name: 'scanStates' }, async (client) => {
                const keys: string[] = []
                let cursor = '0'

                do {
                    const [newCursor, batch] = await client.scan(cursor, 'MATCH', `${REDIS_KEY_STATE}/*`, 'COUNT', 500)
                    cursor = newCursor
                    keys.push(...batch)
                } while (cursor !== '0')

                return keys
            })

        let stateKeys: string[] | null
        try {
            stateKeys = await scan(this.redisReader)
        } catch (err) {
            logger.warn('🔀', '[HogWatcher] reader scanStates failed, falling back to writer', { err })
            stateKeys = await scan(this.redis)
        }

        if (!stateKeys || stateKeys.length === 0) {
            return {}
        }

        // Extract function IDs from the keys
        const functionIds = stateKeys.map((key: string) => key.replace(`${REDIS_KEY_STATE}/`, ''))

        // Get states for all found function IDs
        return await this.getPersistedStates(functionIds)
    }

    public async clearLock(id: HogFunctionType['id']): Promise<void> {
        await this.redis.usePipeline({ name: 'clearLock' }, (pipeline) => {
            pipeline.del(`${REDIS_KEY_STATE_LOCK}/${id}`)
        })
    }

    public async doStageChanges(
        changes: [HogFunctionType, HogWatcherState][],
        forceReset: boolean = false
    ): Promise<void> {
        const res = await this.redis.usePipeline({ name: 'forceStateChange' }, (pipeline) => {
            for (const [hogFunction, state] of changes) {
                hogFunctionStateChange.inc({
                    state: HogWatcherState[state],
                    kind: hogFunction.type,
                })

                const id = hogFunction.id
                const newScore =
                    state === HogWatcherState.healthy
                        ? this.config.bucketSize
                        : state === HogWatcherState.degraded
                          ? this.config.bucketSize * this.config.thresholdDegraded
                          : 0

                const nowSeconds = Math.round(Date.now() / 1000)

                pipeline.getset(`${REDIS_KEY_STATE}/${id}`, state) // Set the state
                pipeline.setex(`${REDIS_KEY_STATE_LOCK}/${id}`, this.config.stateLockTtl, '1') // Set the lock
                if (forceReset) {
                    pipeline.hset(`${REDIS_KEY_TOKENS}/${id}`, 'pool', newScore)
                    pipeline.hset(`${REDIS_KEY_TOKENS}/${id}`, 'ts', nowSeconds)
                }
            }
        })

        if (!res) {
            return
        }

        const numOperations = forceReset ? 4 : 2

        await Promise.all(
            changes.map(async ([hogFunction, state], index) => {
                const [stateResult] = getRedisPipelineResults(res, index, numOperations)
                const previousState = Number(stateResult[1] ?? HogWatcherState.healthy)
                if (previousState !== state) {
                    await this.onStateChange({
                        hogFunction,
                        state,
                        previousState,
                    })
                }
            })
        )
    }

    public async forceStateChange(hogFunction: HogFunctionType, state: HogWatcherState): Promise<void> {
        await this.doStageChanges([[hogFunction, state]])
    }

    public async observeResults(results: CyclotronJobInvocationResult[]): Promise<void> {
        const functionCosts: Record<CyclotronJobInvocation['functionId'], FunctionCostEntry> = {}

        results.forEach((result) => {
            if (!isHogFunctionResult(result)) {
                return
            }

            const functionCost = functionCosts[result.invocation.functionId] ?? {
                functionId: result.invocation.functionId,
                cost: 0,
                hogFunction: result.invocation.hogFunction,
            }

            if (result.finished) {
                // Process each timing entry individually instead of totaling them
                for (const timing of result.invocation.state.timings) {
                    const costConfig = this.costsMapping[timing.kind]
                    if (costConfig) {
                        const ratio =
                            Math.max(timing.duration_ms - costConfig.lowerBound, 0) /
                            (costConfig.upperBound - costConfig.lowerBound)
                        functionCost.cost += Math.round(costConfig.cost * ratio)
                    }
                }
            }

            functionCosts[result.invocation.functionId] = functionCost
        })

        await this.applyCostsAndTransitionStates(Object.values(functionCosts))
    }

    /**
     * Per-(function, Kafka message) cost reporting for log transformations.
     *
     * The events path allocates one CyclotronJobInvocationResult per event; at log-record
     * volumes that is prohibitive, so the logs transformer reports a single aggregated VM
     * duration per function per message instead. Cost is derived from that aggregate via the
     * same piecewise-linear curve (bounds come from this instance's config, so a logs-tuned
     * HogWatcher charges on a logs-appropriate scale). Everything downstream — token bucket,
     * state reads, transition rules — is shared with observeResults.
     */
    public async observeAggregatedResults(
        observations: { hogFunction: HogFunctionType; totalDurationMs: number }[]
    ): Promise<void> {
        const functionCosts: Record<string, FunctionCostEntry> = {}
        const costConfig = this.costsMapping.hog
        if (!costConfig) {
            return
        }

        for (const { hogFunction, totalDurationMs } of observations) {
            const functionCost = functionCosts[hogFunction.id] ?? {
                functionId: hogFunction.id,
                cost: 0,
                hogFunction,
            }
            const ratio =
                Math.max(totalDurationMs - costConfig.lowerBound, 0) / (costConfig.upperBound - costConfig.lowerBound)
            functionCost.cost += Math.round(costConfig.cost * ratio)
            functionCosts[hogFunction.id] = functionCost
        }

        await this.applyCostsAndTransitionStates(Object.values(functionCosts))
    }

    private async applyCostsAndTransitionStates(functionCostEntries: FunctionCostEntry[]): Promise<void> {
        if (functionCostEntries.length === 0) {
            return
        }

        // Split reads (state/lock) to the reader and writes (token bucket) to the writer.
        // These can run concurrently since the reads don't depend on the write results.
        const stateKeys = functionCostEntries.map((fc) => `${REDIS_KEY_STATE}/${fc.functionId}`)
        const lockKeys = functionCostEntries.map((fc) => `${REDIS_KEY_STATE_LOCK}/${fc.functionId}`)

        const readStates = async (pool: RedisV2) => {
            // Single-key pipeline commands avoid CROSSSLOT errors when function IDs map to different cluster slots.
            const results = await pool.usePipeline({ name: 'readStatesForObserve' }, (pipeline) => {
                stateKeys.forEach((key) => pipeline.get(key))
                lockKeys.forEach((key) => pipeline.get(key))
            })
            if (!results) {
                return null
            }
            const commandError = results.find(([error]) => error)?.[0]
            if (commandError) {
                throw commandError
            }
            return {
                states: stateKeys.map((_, index) => results[index]?.[1]),
                locks: lockKeys.map((_, index) => results[stateKeys.length + index]?.[1]),
            }
        }

        const requests = functionCostEntries.map((fc) => ({ id: fc.functionId, cost: fc.cost }))
        const [stateRes, rateLimitRes] = await Promise.all([
            readStates(this.redisReader).catch((err) => {
                logger.warn('🔀', '[HogWatcher] reader readStatesForObserve failed, falling back to writer', { err })
                return readStates(this.redis)
            }),
            this.rateLimiter.rateLimitGrouped(requests),
        ])

        if (!stateRes) {
            return
        }

        const changes: [HogFunctionType, HogWatcherState][] = []

        // Calculate all those that have changed state
        functionCostEntries.map((functionCost, index) => {
            const limit = rateLimitRes[index]?.[1]
            const currentState: HogWatcherState = Number(stateRes.states[index] ?? HogWatcherState.healthy)
            const tokens = Number(limit?.tokens ?? this.config.bucketSize)
            const newState = this.calculateNewState(tokens)

            if (currentState !== newState) {
                if (stateRes.locks[index]) {
                    // We don't want to change the state of a function that is being locked (i.e. recently changed state)
                    return
                }

                if (currentState === HogWatcherState.disabled || currentState >= HogWatcherState.forcefully_degraded) {
                    // We never modify the state of a disabled function automatically, or a forcefully set value
                    return
                }

                if (functionCost.hogFunction) {
                    changes.push([functionCost.hogFunction, newState])
                }
            }
        })

        if (changes.length > 0) {
            await this.doStageChanges(changes)
        }
    }

    public async observeResultsBuffered(result: CyclotronJobInvocationResult): Promise<void> {
        // This can be called a bunch of times and will queue up results to be processed
        // We need to make sure that we only process the results once
        if (!this.queuedResults) {
            let resolvePromise: () => void
            let rejectPromise: (error: unknown) => void
            const promise = new Promise<void>((resolve, reject) => {
                resolvePromise = resolve
                rejectPromise = reject
            })

            this.queuedResults = {
                results: [],
                promise,
                resolve: resolvePromise!,
                reject: rejectPromise!,
                timeout: setTimeout(() => void this.flushBufferedResults(), this.config.observeResultsBufferTimeMs),
            }
        }

        this.queuedResults.results.push(result)
        const bufferedPromise = this.queuedResults.promise

        if (this.queuedResults.results.length >= this.config.observeResultsBufferMaxResults) {
            void this.flushBufferedResults()
        }
        await bufferedPromise
    }

    private async flushBufferedResults(): Promise<void> {
        if (!this.queuedResults) {
            return
        }

        const { results, timeout, resolve, reject } = this.queuedResults
        clearTimeout(timeout)
        this.queuedResults = null
        try {
            await this.observeResults(results)
            resolve()
        } catch (error) {
            reject(error)
        }
    }
}