:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Counter } from 'prom-client'
import { RedisClientPipeline, RedisV2, getRedisPipelineResults } from '~/common/redis/redis-v2'
import { KeyedRateLimiterService } from '~/common/services/keyed-rate-limiter.service'
import { LazyLoader } from '~/common/utils/lazy-loader'
import { logger } from '~/common/utils/logger'
import { captureTeamEvent } from '~/common/utils/posthog'
import { TeamManager } from '~/common/utils/team-manager'
import {
CyclotronJobInvocation,
CyclotronJobInvocationHogFunction,
CyclotronJobInvocationResult,
HogFunctionTiming,
HogFunctionType,
} from '../../types'
export interface HogWatcherConfig {
hogCostTimingLowerMs: number
hogCostTimingUpperMs: number
hogCostTiming: number
asyncCostTimingLowerMs: number
asyncCostTimingUpperMs: number
asyncCostTiming: number
sendEvents: boolean
bucketSize: number
refillRate: number
ttl: number
automaticallyDisableFunctions: boolean
thresholdDegraded: number
stateLockTtl: number
observeResultsBufferTimeMs: number
observeResultsBufferMaxResults: number
}
export const BASE_REDIS_KEY = process.env.NODE_ENV == 'test' ? '@posthog-test/hog-watcher-2' : '@posthog/hog-watcher-2'
const REDIS_KEY_TOKENS = `${BASE_REDIS_KEY}/tokens`
const REDIS_KEY_STATE = `${BASE_REDIS_KEY}/state`
const REDIS_KEY_STATE_LOCK = `${BASE_REDIS_KEY}/state-lock`
export enum HogWatcherState {
healthy = 1,
degraded = 2,
disabled = 3,
// These are states that we do not auto transition into - can only be modified by the admin tool
forcefully_degraded = 11,
forcefully_disabled = 12,
}
export type HogWatcherFunctionState = {
tokens: number
state: HogWatcherState
}
/**
* Compares a watcher read across Redis and Valkey on `state` alone, for `dualRead`.
*
* `tokens` is refilled from the wall clock at read time (see `getPersistedStates`), so the two
* stores disagree on it whenever their reads land on different seconds. Comparing it would leave
* the mismatch metric permanently saturated, and that metric is the signal we need to trust
* before moving watcher reads to Valkey. `state` is what decides whether an invocation runs.
*/
export function sameWatcherState(
primary: HogWatcherFunctionState | null,
secondary: HogWatcherFunctionState | null
): boolean {
return primary?.state === secondary?.state
}
/** `sameWatcherState` over a keyed batch — the id set has to agree as well as each state. */
export function sameWatcherStates(
primary: Record<string, HogWatcherFunctionState>,
secondary: Record<string, HogWatcherFunctionState>
): boolean {
const ids = Object.keys(primary ?? {})
return (
ids.length === Object.keys(secondary ?? {}).length &&
ids.every((id) => primary[id]?.state === secondary[id]?.state)
)
}
type FunctionCostEntry = {
hogFunction?: HogFunctionType
functionId: string
cost: number
}
const hogFunctionStateChange = new Counter({
name: 'cdp_hog_function_state_change',
help: 'Number of times a transformation state changed',
labelNames: ['state', 'kind'],
})
type HogFunctionTimingCost = {
lowerBound: number
upperBound: number
cost: number
}
type HogFunctionTimingCosts = Partial<Record<HogFunctionTiming['kind'], HogFunctionTimingCost>>
// Check if the result is of type CyclotronJobInvocationHogFunction
export const isHogFunctionResult = (
result: CyclotronJobInvocationResult
): result is CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> => {
return 'hogFunction' in result.invocation
}
// Helper if you don't care about the forced side of things
export const effectiveState = (state: HogWatcherState) => {
if (state === HogWatcherState.forcefully_degraded) {
return HogWatcherState.degraded
}
if (state === HogWatcherState.forcefully_disabled) {
return HogWatcherState.disabled
}
return state
}
export class HogWatcherService {
private costsMapping: HogFunctionTimingCosts
private lazyLoader: LazyLoader<HogWatcherFunctionState>
private queuedResults: {
results: CyclotronJobInvocationResult[]
promise: Promise<void>
timeout: NodeJS.Timeout
resolve: () => void
reject: (error: unknown) => void
} | null = null
private redisReader: RedisV2
private rateLimiter: KeyedRateLimiterService
constructor(
private teamManager: TeamManager,
private config: HogWatcherConfig,
private redis: RedisV2,
redisReader?: RedisV2
) {
this.redisReader = redisReader ?? redis
// Token-bucket rate limiter — `name: 'hog-watcher-2'` produces the same
// Redis key prefix this service has used historically (matches BASE_REDIS_KEY).
this.rateLimiter = new KeyedRateLimiterService(
{
name: 'hog-watcher-2',
bucketSize: this.config.bucketSize,
refillRate: this.config.refillRate,
ttlSeconds: this.config.ttl,
},
this.redis
)
this.costsMapping = {
hog: {
lowerBound: this.config.hogCostTimingLowerMs,
upperBound: this.config.hogCostTimingUpperMs,
cost: this.config.hogCostTiming,
},
async_function: {
lowerBound: this.config.asyncCostTimingLowerMs,
upperBound: this.config.asyncCostTimingUpperMs,
cost: this.config.asyncCostTiming,
},
}
for (const [kind, mapping] of Object.entries(this.costsMapping)) {
if (mapping.lowerBound >= mapping.upperBound) {
throw new Error(
`Lower bound for kind ${kind} of ${mapping.lowerBound}ms must be lower than upper bound of ${mapping.upperBound}ms. This is a configuration error.`
)
}
}
this.lazyLoader = new LazyLoader({
name: 'hog_watcher_lazy_loader',
refreshAgeMs: 30_000, // Cache for 30 seconds
refreshJitterMs: 10_000,
loader: async (ids) => await this.getPersistedStates(ids),
})
}
private async onStateChange({
hogFunction,
state,
previousState,
}: {
hogFunction: HogFunctionType
state: HogWatcherState
previousState: HogWatcherState
}) {
const team = await this.teamManager.getTeam(hogFunction.team_id)
logger.info('[HogWatcherService] onStateChange', {
hogFunctionId: hogFunction.id,
hogFunctionName: hogFunction.name,
state,
previousState,
})
if (team && this.config.sendEvents) {
captureTeamEvent(team, 'hog_function_state_change', {
hog_function_id: hogFunction.id,
hog_function_type: hogFunction.type,
hog_function_name: hogFunction.name,
hog_function_template_id: hogFunction.template_id,
state: HogWatcherState[state], // Convert numeric state to readable string
previous_state: HogWatcherState[previousState], // Convert numeric state to readable string
})
}
}
public calculateNewState(tokens: number): HogWatcherState {
const rating = tokens / this.config.bucketSize
if (rating < 0 && this.config.automaticallyDisableFunctions) {
return HogWatcherState.disabled
}
if (rating <= this.config.thresholdDegraded) {
return HogWatcherState.degraded
}
return HogWatcherState.healthy
}
/**
* Get the persisted states of a list of hog functions.
*
* Uses plain hget calls instead of the evalsha Lua script.
* The Lua script with cost=0 was doing 2 hget + 2 hset + 1 expire
* per call — all writes unnecessary for a read-only check.
* Replacing with 2 hget calls eliminates ~60% of internal Redis
* operations per read and makes this method safe to route to
* read replicas in the future.
*/
public async getPersistedStates(
ids: HogFunctionType['id'][]
): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
const idsSet = new Set(ids)
const nowSeconds = Math.round(Date.now() / 1000)
const buildGetStatesPipeline = (pipeline: RedisClientPipeline) => {
for (const id of idsSet) {
pipeline.hmget(`${REDIS_KEY_TOKENS}/${id}`, 'pool', 'ts')
pipeline.get(`${REDIS_KEY_STATE}/${id}`)
}
}
let res
try {
res = await this.redisReader.usePipeline({ name: 'getStates' }, buildGetStatesPipeline)
} catch (err) {
logger.warn('🔀', '[HogWatcher] reader getStates failed, falling back to writer', { err })
res = await this.redis.usePipeline({ name: 'getStates' }, buildGetStatesPipeline)
}
return Array.from(idsSet).reduce(
(acc, id, index) => {
const resIndex = index * 2
const [pool, ts] = res?.[resIndex]?.[1] ?? [null, null]
const stateVal = res?.[resIndex + 1]?.[1]
// Same refill calculation as the Lua token bucket script
let tokens: number
if (pool === null || pool === undefined) {
tokens = this.config.bucketSize
} else {
const timeDiff = ts ? Math.max(nowSeconds - Number(ts), 0) : 0
const owedTokens = timeDiff * this.config.refillRate
tokens = Math.min(Number(pool) + owedTokens, this.config.bucketSize)
}
acc[id] = {
state: stateVal ? Number(stateVal) : HogWatcherState.healthy,
tokens,
}
return acc
},
{} as Record<HogFunctionType['id'], HogWatcherFunctionState>
)
}
/**
* Like getPersistedStates but returns the state of a single hog function
*/
public async getPersistedState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState> {
const res = await this.getPersistedStates([id])
return res[id]
}
public async getCachedPersistedState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState | null> {
return await this.lazyLoader.get(id)
}
/**
* Like getPersistedStates but returns the effective state (i.e. ignores forcefully set states)
*/
public async getEffectiveStates(
ids: HogFunctionType['id'][]
): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
const states = await this.getPersistedStates(ids)
return Object.fromEntries(
Object.entries(states).map(([id, state]) => [
id,
{ state: effectiveState(state.state), tokens: state.tokens },
])
)
}
/**
* Like getPersistedState but returns the effective state (i.e. ignores forcefully set states)
*/
public async getEffectiveState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState> {
const res = await this.getEffectiveStates([id])
return res[id]
}
public async getCachedEffectiveState(id: HogFunctionType['id']): Promise<HogWatcherFunctionState | null> {
const res = await this.lazyLoader.get(id)
if (!res) {
return null
}
return { state: effectiveState(res.state), tokens: res.tokens }
}
public async getAllFunctionStates(): Promise<Record<HogFunctionType['id'], HogWatcherFunctionState>> {
const scan = (pool: RedisV2): Promise<string[] | null> =>
pool.useClient({ name: 'scanStates' }, async (client) => {
const keys: string[] = []
let cursor = '0'
do {
const [newCursor, batch] = await client.scan(cursor, 'MATCH', `${REDIS_KEY_STATE}/*`, 'COUNT', 500)
cursor = newCursor
keys.push(...batch)
} while (cursor !== '0')
return keys
})
let stateKeys: string[] | null
try {
stateKeys = await scan(this.redisReader)
} catch (err) {
logger.warn('🔀', '[HogWatcher] reader scanStates failed, falling back to writer', { err })
stateKeys = await scan(this.redis)
}
if (!stateKeys || stateKeys.length === 0) {
return {}
}
// Extract function IDs from the keys
const functionIds = stateKeys.map((key: string) => key.replace(`${REDIS_KEY_STATE}/`, ''))
// Get states for all found function IDs
return await this.getPersistedStates(functionIds)
}
public async clearLock(id: HogFunctionType['id']): Promise<void> {
await this.redis.usePipeline({ name: 'clearLock' }, (pipeline) => {
pipeline.del(`${REDIS_KEY_STATE_LOCK}/${id}`)
})
}
public async doStageChanges(
changes: [HogFunctionType, HogWatcherState][],
forceReset: boolean = false
): Promise<void> {
const res = await this.redis.usePipeline({ name: 'forceStateChange' }, (pipeline) => {
for (const [hogFunction, state] of changes) {
hogFunctionStateChange.inc({
state: HogWatcherState[state],
kind: hogFunction.type,
})
const id = hogFunction.id
const newScore =
state === HogWatcherState.healthy
? this.config.bucketSize
: state === HogWatcherState.degraded
? this.config.bucketSize * this.config.thresholdDegraded
: 0
const nowSeconds = Math.round(Date.now() / 1000)
pipeline.getset(`${REDIS_KEY_STATE}/${id}`, state) // Set the state
pipeline.setex(`${REDIS_KEY_STATE_LOCK}/${id}`, this.config.stateLockTtl, '1') // Set the lock
if (forceReset) {
pipeline.hset(`${REDIS_KEY_TOKENS}/${id}`, 'pool', newScore)
pipeline.hset(`${REDIS_KEY_TOKENS}/${id}`, 'ts', nowSeconds)
}
}
})
if (!res) {
return
}
const numOperations = forceReset ? 4 : 2
await Promise.all(
changes.map(async ([hogFunction, state], index) => {
const [stateResult] = getRedisPipelineResults(res, index, numOperations)
const previousState = Number(stateResult[1] ?? HogWatcherState.healthy)
if (previousState !== state) {
await this.onStateChange({
hogFunction,
state,
previousState,
})
}
})
)
}
public async forceStateChange(hogFunction: HogFunctionType, state: HogWatcherState): Promise<void> {
await this.doStageChanges([[hogFunction, state]])
}
public async observeResults(results: CyclotronJobInvocationResult[]): Promise<void> {
const functionCosts: Record<CyclotronJobInvocation['functionId'], FunctionCostEntry> = {}
results.forEach((result) => {
if (!isHogFunctionResult(result)) {
return
}
const functionCost = functionCosts[result.invocation.functionId] ?? {
functionId: result.invocation.functionId,
cost: 0,
hogFunction: result.invocation.hogFunction,
}
if (result.finished) {
// Process each timing entry individually instead of totaling them
for (const timing of result.invocation.state.timings) {
const costConfig = this.costsMapping[timing.kind]
if (costConfig) {
const ratio =
Math.max(timing.duration_ms - costConfig.lowerBound, 0) /
(costConfig.upperBound - costConfig.lowerBound)
functionCost.cost += Math.round(costConfig.cost * ratio)
}
}
}
functionCosts[result.invocation.functionId] = functionCost
})
await this.applyCostsAndTransitionStates(Object.values(functionCosts))
}
/**
* Per-(function, Kafka message) cost reporting for log transformations.
*
* The events path allocates one CyclotronJobInvocationResult per event; at log-record
* volumes that is prohibitive, so the logs transformer reports a single aggregated VM
* duration per function per message instead. Cost is derived from that aggregate via the
* same piecewise-linear curve (bounds come from this instance's config, so a logs-tuned
* HogWatcher charges on a logs-appropriate scale). Everything downstream — token bucket,
* state reads, transition rules — is shared with observeResults.
*/
public async observeAggregatedResults(
observations: { hogFunction: HogFunctionType; totalDurationMs: number }[]
): Promise<void> {
const functionCosts: Record<string, FunctionCostEntry> = {}
const costConfig = this.costsMapping.hog
if (!costConfig) {
return
}
for (const { hogFunction, totalDurationMs } of observations) {
const functionCost = functionCosts[hogFunction.id] ?? {
functionId: hogFunction.id,
cost: 0,
hogFunction,
}
const ratio =
Math.max(totalDurationMs - costConfig.lowerBound, 0) / (costConfig.upperBound - costConfig.lowerBound)
functionCost.cost += Math.round(costConfig.cost * ratio)
functionCosts[hogFunction.id] = functionCost
}
await this.applyCostsAndTransitionStates(Object.values(functionCosts))
}
private async applyCostsAndTransitionStates(functionCostEntries: FunctionCostEntry[]): Promise<void> {
if (functionCostEntries.length === 0) {
return
}
// Split reads (state/lock) to the reader and writes (token bucket) to the writer.
// These can run concurrently since the reads don't depend on the write results.
const stateKeys = functionCostEntries.map((fc) => `${REDIS_KEY_STATE}/${fc.functionId}`)
const lockKeys = functionCostEntries.map((fc) => `${REDIS_KEY_STATE_LOCK}/${fc.functionId}`)
const readStates = async (pool: RedisV2) => {
// Single-key pipeline commands avoid CROSSSLOT errors when function IDs map to different cluster slots.
const results = await pool.usePipeline({ name: 'readStatesForObserve' }, (pipeline) => {
stateKeys.forEach((key) => pipeline.get(key))
lockKeys.forEach((key) => pipeline.get(key))
})
if (!results) {
return null
}
const commandError = results.find(([error]) => error)?.[0]
if (commandError) {
throw commandError
}
return {
states: stateKeys.map((_, index) => results[index]?.[1]),
locks: lockKeys.map((_, index) => results[stateKeys.length + index]?.[1]),
}
}
const requests = functionCostEntries.map((fc) => ({ id: fc.functionId, cost: fc.cost }))
const [stateRes, rateLimitRes] = await Promise.all([
readStates(this.redisReader).catch((err) => {
logger.warn('🔀', '[HogWatcher] reader readStatesForObserve failed, falling back to writer', { err })
return readStates(this.redis)
}),
this.rateLimiter.rateLimitGrouped(requests),
])
if (!stateRes) {
return
}
const changes: [HogFunctionType, HogWatcherState][] = []
// Calculate all those that have changed state
functionCostEntries.map((functionCost, index) => {
const limit = rateLimitRes[index]?.[1]
const currentState: HogWatcherState = Number(stateRes.states[index] ?? HogWatcherState.healthy)
const tokens = Number(limit?.tokens ?? this.config.bucketSize)
const newState = this.calculateNewState(tokens)
if (currentState !== newState) {
if (stateRes.locks[index]) {
// We don't want to change the state of a function that is being locked (i.e. recently changed state)
return
}
if (currentState === HogWatcherState.disabled || currentState >= HogWatcherState.forcefully_degraded) {
// We never modify the state of a disabled function automatically, or a forcefully set value
return
}
if (functionCost.hogFunction) {
changes.push([functionCost.hogFunction, newState])
}
}
})
if (changes.length > 0) {
await this.doStageChanges(changes)
}
}
public async observeResultsBuffered(result: CyclotronJobInvocationResult): Promise<void> {
// This can be called a bunch of times and will queue up results to be processed
// We need to make sure that we only process the results once
if (!this.queuedResults) {
let resolvePromise: () => void
let rejectPromise: (error: unknown) => void
const promise = new Promise<void>((resolve, reject) => {
resolvePromise = resolve
rejectPromise = reject
})
this.queuedResults = {
results: [],
promise,
resolve: resolvePromise!,
reject: rejectPromise!,
timeout: setTimeout(() => void this.flushBufferedResults(), this.config.observeResultsBufferTimeMs),
}
}
this.queuedResults.results.push(result)
const bufferedPromise = this.queuedResults.promise
if (this.queuedResults.results.length >= this.config.observeResultsBufferMaxResults) {
void this.flushBufferedResults()
}
await bufferedPromise
}
private async flushBufferedResults(): Promise<void> {
if (!this.queuedResults) {
return
}
const { results, timeout, resolve, reject } = this.queuedResults
clearTimeout(timeout)
this.queuedResults = null
try {
await this.observeResults(results)
resolve()
} catch (error) {
reject(error)
}
}
}