Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.5 KB
Raw Blame
/**
 * Evaluation Scheduler
 *
 * Consumes AI events (e.g., $ai_generation) from the ai_events Kafka topic
 * (clickhouse_ai_events_json), matches them against evaluation filters, and
 * triggers evaluation workflows via Temporal when conditions are met.
 *
 * The ai_events topic carries the unstripped event payload (the events topic
 * has the heavy AI properties stripped), so the judge always sees the full
 * input/output it needs to grade.
 */
import * as crypto from 'crypto'
import { Message } from 'node-rdkafka'
import { Counter } from 'prom-client'

import { AIObservabilityConfig } from '~/ai-observability/config'
import { EvaluationManagerService } from '~/ai-observability/services/evaluation-manager.service'
import { ProviderKeyManagerService } from '~/ai-observability/services/provider-key-manager.service'
import { TaggerManagerService } from '~/ai-observability/services/tagger-manager.service'
import {
    TemporalService,
    TemporalServiceConfig,
    isEvaluationWorkflowRuntime,
    resolveSettleConfig,
} from '~/ai-observability/services/temporal.service'
import { Evaluation, EvaluationConditionSet, Matchable, Tagger } from '~/ai-observability/types'
import { execHog } from '~/cdp/utils/hog-exec'
import { KAFKA_CLICKHOUSE_AI_EVENTS_JSON, prefix as KAFKA_PREFIX } from '~/common/config/kafka-topics'
import { createKafkaConsumer } from '~/common/kafka/consumer'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { PubSub } from '~/common/utils/pubsub'
import { PluginServerService, RawKafkaEvent } from '~/types'

export type EvaluationSchedulerConfig = TemporalServiceConfig &
    Pick<AIObservabilityConfig, 'LLMA_EVAL_SCHEDULER_PROVIDER_KEY_GATING'>

export interface EvaluationSchedulerDeps {
    postgres: PostgresRouter
    pubSub: PubSub
}

const evaluationSchedulerEventsProcessed = new Counter({
    name: 'evaluation_scheduler_events_processed',
    help: 'Number of AI events processed by evaluation scheduler',
    labelNames: ['status', 'type'], // type: 'evaluation' | 'tagger' — keep existing eval alerts working while exposing tagger traffic
})

const evaluationMatchesCounter = new Counter({
    name: 'evaluation_matches',
    help: 'Number of evaluation matches by outcome',
    labelNames: ['outcome', 'type'], // matched, filtered, sampling_excluded, error × evaluation/tagger; no_ai_session_id on evaluation only
})

const evaluationSchedulerMessagesReceived = new Counter({
    name: 'evaluation_scheduler_messages_received',
    help: 'Number of Kafka messages received before filtering',
})

const evaluationSchedulerEventsFiltered = new Counter({
    name: 'evaluation_scheduler_events_filtered',
    help: 'Number of events after productTrack header filter',
    labelNames: ['passed'],
})

const evaluationSchedulerHeaderValues = new Counter({
    name: 'evaluation_scheduler_header_values',
    help: 'Count of different productTrack header values seen',
    labelNames: ['header_value'],
})

// Pure functions for testability

/**
 * Pull the value out of an awaited Promise.allSettled entry, falling back to an
 * empty object map and logging the rejection reason. Used to keep the scheduler
 * loop alive when one fetch fails without losing the other one's results.
 */
export function unwrapOrLog<T extends Record<string, unknown>>(
    result: PromiseSettledResult<T>,
    errorMessage: string
): T {
    if (result.status === 'fulfilled') {
        return result.value
    }
    logger.error(errorMessage, {
        error: result.reason instanceof Error ? result.reason.message : String(result.reason),
    })
    return {} as T
}

export function filterAndParseMessages(messages: Message[]): RawKafkaEvent[] {
    return messages
        .filter((message) => {
            const headers = message.headers as { productTrack?: Buffer }[] | undefined
            const productTrack = headers?.find((h) => h.productTrack)?.productTrack?.toString('utf8')

            evaluationSchedulerHeaderValues.labels({ header_value: productTrack || 'missing' }).inc()

            return productTrack === 'llma'
        })
        .map((message) => {
            try {
                return parseJSON(message.value!.toString()) as RawKafkaEvent
            } catch (e) {
                logger.error('Error parsing event', { error: e })
                return null
            }
        })
        .filter((event): event is RawKafkaEvent => event !== null)
        .filter((event) => event.event === '$ai_generation')
}

export function groupEventsByTeam(events: RawKafkaEvent[]): Map<number, RawKafkaEvent[]> {
    const grouped = new Map<number, RawKafkaEvent[]>()
    for (const event of events) {
        const teamEvents = grouped.get(event.team_id) || []
        teamEvents.push(event)
        grouped.set(event.team_id, teamEvents)
    }
    return grouped
}

/**
 * Pull the evaluation-unit linkage out of an event's properties. Ids are user-controlled and can
 * be ingested as numbers (e.g. a buggy `trace_id: 0`), so values are string-coerced; empty or
 * missing ids resolve to null.
 *
 * `sessionId` ($session_id) is PostHog's product-analytics session and only ever travels through
 * to the emitted evaluation event. `aiSessionId` ($ai_session_id) is the AI-observability session
 * and is the unit a session-target evaluation runs on. They are different id spaces.
 */
export function extractEvaluationContext(event: RawKafkaEvent): {
    traceId: string | null
    sessionId: string | null
    aiSessionId: string | null
} {
    let properties: Record<string, unknown> = {}
    try {
        properties = parseJSON(event.properties || '{}')
    } catch {
        return { traceId: null, sessionId: null, aiSessionId: null }
    }
    const coerce = (value: unknown): string | null =>
        value === null || value === undefined || value === '' ? null : String(value)
    return {
        traceId: coerce(properties['$ai_trace_id']),
        sessionId: coerce(properties['$session_id']),
        aiSessionId: coerce(properties['$ai_session_id']),
    }
}

export function checkRolloutPercentage(eventId: string, rolloutPercentage: number): boolean {
    if (rolloutPercentage >= 100) {
        return true
    }

    // Use MD5 hash for deterministic sampling
    const hash = crypto.createHash('md5').update(eventId).digest('hex')
    const hashValue = parseInt(hash.substring(0, 8), 16)
    const percentage = (hashValue % 10000) / 100

    return percentage < rolloutPercentage
}

export async function checkConditionMatch(event: RawKafkaEvent, condition: EvaluationConditionSet): Promise<boolean> {
    if (!condition.bytecode) {
        if (condition.bytecode_error) {
            logger.warn('Condition has bytecode error, skipping', {
                conditionId: condition.id,
                error: condition.bytecode_error,
            })
        }
        return false
    }

    // Build globals for HogVM execution
    let personProperties = {}
    let eventProperties = {}

    try {
        personProperties = parseJSON(event.person_properties || '{}')
    } catch (e) {
        logger.warn('Failed to parse person_properties', {
            conditionId: condition.id,
            eventUuid: event.uuid,
            error: e instanceof Error ? e.message : String(e),
        })
    }

    try {
        eventProperties = parseJSON(event.properties || '{}')
    } catch (e) {
        logger.warn('Failed to parse event properties', {
            conditionId: condition.id,
            eventUuid: event.uuid,
            error: e instanceof Error ? e.message : String(e),
        })
    }

    const filterGlobals = {
        event: event.event,
        elements_chain: event.elements_chain || '',
        distinct_id: event.distinct_id,
        person: {
            properties: personProperties,
        },
        properties: eventProperties,
    }

    try {
        const execResult = await execHog(condition.bytecode, { globals: filterGlobals })

        if (execResult.error || execResult.execResult?.error) {
            logger.error('Error executing bytecode', {
                conditionId: condition.id,
                error: execResult.error ?? execResult.execResult?.error,
            })
            return false
        }

        return typeof execResult.execResult?.result === 'boolean' && execResult.execResult.result
    } catch (error: unknown) {
        logger.error('Exception executing bytecode', {
            conditionId: condition.id,
            error: error instanceof Error ? error.message : String(error),
        })
        return false
    }
}

export type EvaluationMatchResult =
    | { matched: true; conditionId: string }
    | { matched: false; reason: 'no_conditions' | 'disabled' | 'filtered' | 'sampling_excluded' }

export class EvaluationMatcher {
    /**
     * `samplingKey` defaults to the event uuid (independent coin flip per generation). Trace-
     * target evals pass the trace id instead so the whole trace is atomically in or out of the
     * sample, no matter which of its generations is seen first.
     */
    async shouldTriggerEvaluation(
        event: RawKafkaEvent,
        evaluation: Matchable,
        samplingKey?: string
    ): Promise<EvaluationMatchResult> {
        if (!evaluation.enabled) {
            return { matched: false, reason: 'disabled' }
        }

        const conditions = evaluation.conditions
        if (conditions.length === 0) {
            return { matched: false, reason: 'no_conditions' }
        }

        for (const condition of conditions) {
            const conditionMatched = await checkConditionMatch(event, condition)

            if (!conditionMatched) {
                continue
            }

            const inSample = checkRolloutPercentage(samplingKey ?? event.uuid, condition.rollout_percentage)

            if (!inSample) {
                continue
            }

            return { matched: true, conditionId: condition.id }
        }

        return { matched: false, reason: 'filtered' }
    }
}

export const startEvaluationScheduler = async (
    config: EvaluationSchedulerConfig,
    deps: EvaluationSchedulerDeps
): Promise<PluginServerService> => {
    const groupId = `${KAFKA_PREFIX}evaluation-scheduler-ai-events`
    const kafkaTopic = KAFKA_CLICKHOUSE_AI_EVENTS_JSON

    logger.info('🤖', 'Starting evaluation scheduler', {
        groupId,
        kafkaTopic,
        providerKeyGating: config.LLMA_EVAL_SCHEDULER_PROVIDER_KEY_GATING,
    })

    const temporalService = new TemporalService(config)
    const evaluationManager = new EvaluationManagerService(deps.postgres, deps.pubSub)
    const taggerManager = new TaggerManagerService(deps.postgres, deps.pubSub)
    const providerKeyManager = config.LLMA_EVAL_SCHEDULER_PROVIDER_KEY_GATING
        ? new ProviderKeyManagerService(deps.postgres, deps.pubSub)
        : undefined

    const kafkaConsumer = createKafkaConsumer({
        groupId,
        topic: kafkaTopic,
    })

    await kafkaConsumer.connect((messages) =>
        eachBatchEvaluationScheduler(messages, evaluationManager, taggerManager, temporalService, {
            enabled: config.LLMA_EVAL_SCHEDULER_PROVIDER_KEY_GATING,
            providerKeyManager,
        })
    )

    const onShutdown = async (): Promise<void> => {
        await temporalService.disconnect()
        await kafkaConsumer.disconnect()
    }

    return {
        id: 'evaluation-scheduler',
        healthcheck: () => kafkaConsumer.isHealthy(),
        onShutdown,
    }
}

export interface ProviderKeyGateOptions {
    enabled: boolean
    providerKeyManager?: Pick<ProviderKeyManagerService, 'getProviderKey'>
}

type ProviderKeyGateDefinition = {
    id: string
    team_id: number
    provider_key_id?: string | null
    evaluation_type?: string
    tagger_type?: string
}

const NON_PROVIDER_KEY_RUNTIMES = new Set(['hog', 'sentiment'])

function definitionUsesProviderKey(definition: ProviderKeyGateDefinition): boolean {
    const runtime = definition.evaluation_type ?? definition.tagger_type
    return !runtime || !NON_PROVIDER_KEY_RUNTIMES.has(runtime)
}

export async function eachBatchEvaluationScheduler(
    messages: Message[],
    evaluationManager: EvaluationManagerService,
    taggerManager: TaggerManagerService,
    temporalService: TemporalService,
    providerKeyGate?: ProviderKeyGateOptions
): Promise<void> {
    logger.debug('Processing batch', { messageCount: messages.length })

    evaluationSchedulerMessagesReceived.inc(messages.length)

    const aiGenerationEvents = filterAndParseMessages(messages)

    evaluationSchedulerEventsFiltered.labels({ passed: 'false' }).inc(messages.length - aiGenerationEvents.length)
    evaluationSchedulerEventsFiltered.labels({ passed: 'true' }).inc(aiGenerationEvents.length)

    logger.debug('Filtered batch', {
        totalMessages: messages.length,
        aiEventsFound: aiGenerationEvents.length,
        filteredOut: messages.length - aiGenerationEvents.length,
    })

    if (aiGenerationEvents.length === 0) {
        return
    }

    logger.debug('Found $ai_generation events', { count: aiGenerationEvents.length })

    const eventsByTeam = groupEventsByTeam(aiGenerationEvents)
    const teamIds = Array.from(eventsByTeam.keys())

    // Fetch evaluations and taggers independently — a transient DB failure on one
    // side must not take down dispatch for the other. If either side fails we log
    // and fall back to an empty map so the matched workflows just don't trigger
    // this batch, instead of failing the whole Kafka consumer loop.
    const [evaluationsResult, taggersResult] = await Promise.allSettled([
        evaluationManager.getEvaluationsForTeams(teamIds),
        taggerManager.getTaggersForTeams(teamIds),
    ])
    const evaluationsByTeam = unwrapOrLog(evaluationsResult, 'Failed to fetch evaluations for teams')
    const taggersByTeam = unwrapOrLog(taggersResult, 'Failed to fetch taggers for teams')
    const matcher = new EvaluationMatcher()
    const tasks: Promise<void>[] = []

    for (const [teamId, events] of eventsByTeam.entries()) {
        const evaluationDefinitions = evaluationsByTeam[teamId] || []
        const taggerDefinitions = taggersByTeam[teamId] || []

        if (evaluationDefinitions.length === 0 && taggerDefinitions.length === 0) {
            continue
        }

        for (const event of events) {
            for (const evaluationDefinition of evaluationDefinitions) {
                const task = processEventEvaluationMatch(
                    event,
                    evaluationDefinition,
                    matcher,
                    temporalService,
                    providerKeyGate
                ).catch((error: unknown) => {
                    logger.error('Error processing evaluation', {
                        evaluationId: evaluationDefinition.id,
                        eventUuid: event.uuid,
                        error: error instanceof Error ? error.message : String(error),
                    })
                    evaluationMatchesCounter.labels({ outcome: 'error', type: 'evaluation' }).inc()
                })

                tasks.push(task)
            }

            for (const taggerDefinition of taggerDefinitions) {
                const task = processEventTaggerMatch(
                    event,
                    taggerDefinition,
                    matcher,
                    temporalService,
                    providerKeyGate
                ).catch((error: unknown) => {
                    logger.error('Error processing tagger', {
                        taggerId: taggerDefinition.id,
                        eventUuid: event.uuid,
                        error: error instanceof Error ? error.message : String(error),
                    })
                    evaluationMatchesCounter.labels({ outcome: 'error', type: 'tagger' }).inc()
                })

                tasks.push(task)
            }
        }
    }

    await Promise.allSettled(tasks)

    logger.debug('Batch processing complete', {
        teamsProcessed: eventsByTeam.size,
        totalChecks: tasks.length,
    })
}

async function processEventEvaluationMatch(
    event: RawKafkaEvent,
    evaluationDefinition: Evaluation,
    matcher: EvaluationMatcher,
    temporalService: TemporalService,
    providerKeyGate?: ProviderKeyGateOptions
): Promise<void> {
    evaluationSchedulerEventsProcessed.labels({ status: 'received', type: 'evaluation' }).inc()

    const target = evaluationDefinition.target
    const isAggregateTarget = target === 'trace' || target === 'session'
    let context: ReturnType<typeof extractEvaluationContext> | null = null
    if (isAggregateTarget) {
        context = extractEvaluationContext(event)
        if (!context.traceId) {
            evaluationMatchesCounter.labels({ outcome: 'no_trace_id', type: 'evaluation' }).inc()
            return
        }
        if (target === 'session' && !context.aiSessionId) {
            // Expected for any producer that doesn't set $ai_session_id (most of them). This
            // counter is the coverage signal for whether session evals can fire at all.
            evaluationMatchesCounter.labels({ outcome: 'no_ai_session_id', type: 'evaluation' }).inc()
            return
        }
    }

    // Sample per unit so a whole trace, or a whole session, is atomically in or out.
    const samplingKey = target === 'session' ? context?.aiSessionId : context?.traceId
    const result = await matcher.shouldTriggerEvaluation(event, evaluationDefinition, samplingKey ?? undefined)

    if (!result.matched) {
        evaluationMatchesCounter.labels({ outcome: result.reason, type: 'evaluation' }).inc()
        return
    }

    const providerKeySkipOutcome = await getProviderKeySkipOutcome(evaluationDefinition, providerKeyGate)
    if (providerKeySkipOutcome) {
        evaluationMatchesCounter.labels({ outcome: providerKeySkipOutcome, type: 'evaluation' }).inc()
        return
    }

    logger.debug('Evaluation matched, enqueueing evaluation run', {
        evaluationId: evaluationDefinition.id,
        eventUuid: event.uuid,
        traceId: context?.traceId,
        aiSessionId: context?.aiSessionId,
        conditionId: result.conditionId,
    })

    evaluationMatchesCounter.labels({ outcome: 'matched', type: 'evaluation' }).inc()

    if (isAggregateTarget && context?.traceId) {
        // No isEvaluationWorkflowRuntime guard here: the aggregate workflow validates evaluation_type
        // server-side and rejects unsupported types as a non-retryable ApplicationError.
        await temporalService.startAggregateEvaluationWorkflow({
            evaluationId: evaluationDefinition.id,
            event,
            target,
            traceId: context.traceId,
            sessionId: context.sessionId,
            aiSessionId: context.aiSessionId,
            settle: resolveSettleConfig(evaluationDefinition.target_config, target),
        })
    } else {
        const evaluationRuntime = evaluationDefinition.evaluation_type
        if (!isEvaluationWorkflowRuntime(evaluationRuntime)) {
            throw new Error(`Unsupported evaluation runtime: ${evaluationRuntime}`)
        }

        await temporalService.startEvaluationRunWorkflow(evaluationDefinition.id, event, evaluationRuntime)
    }
    evaluationSchedulerEventsProcessed.labels({ status: 'success', type: 'evaluation' }).inc()
}

async function processEventTaggerMatch(
    event: RawKafkaEvent,
    taggerDefinition: Tagger,
    matcher: EvaluationMatcher,
    temporalService: TemporalService,
    providerKeyGate?: ProviderKeyGateOptions
): Promise<void> {
    evaluationSchedulerEventsProcessed.labels({ status: 'received', type: 'tagger' }).inc()

    // Taggers use the same conditions structure as evaluations
    const result = await matcher.shouldTriggerEvaluation(event, taggerDefinition)

    if (!result.matched) {
        evaluationMatchesCounter.labels({ outcome: result.reason, type: 'tagger' }).inc()
        return
    }

    const providerKeySkipOutcome = await getProviderKeySkipOutcome(taggerDefinition, providerKeyGate)
    if (providerKeySkipOutcome) {
        evaluationMatchesCounter.labels({ outcome: providerKeySkipOutcome, type: 'tagger' }).inc()
        return
    }

    logger.debug('Tagger matched, enqueueing tagger run', {
        taggerId: taggerDefinition.id,
        eventUuid: event.uuid,
        conditionId: result.conditionId,
    })

    evaluationMatchesCounter.labels({ outcome: 'matched', type: 'tagger' }).inc()

    await temporalService.startTaggerRunWorkflow(taggerDefinition.id, event)
    evaluationSchedulerEventsProcessed.labels({ status: 'success', type: 'tagger' }).inc()
}

async function getProviderKeySkipOutcome(
    definition: ProviderKeyGateDefinition,
    providerKeyGate?: ProviderKeyGateOptions
): Promise<'provider_key_not_ok' | 'provider_key_not_found' | 'provider_key_team_mismatch' | null> {
    if (!providerKeyGate?.enabled || !providerKeyGate.providerKeyManager) {
        return null
    }

    if (!definitionUsesProviderKey(definition)) {
        return null
    }

    const providerKeyId = definition.provider_key_id
    if (!providerKeyId) {
        return null
    }

    try {
        const providerKey = await providerKeyGate.providerKeyManager.getProviderKey(providerKeyId)
        if (!providerKey) {
            return 'provider_key_not_found'
        }
        if (providerKey.team_id !== definition.team_id) {
            return 'provider_key_team_mismatch'
        }
        if (providerKey.state !== 'ok') {
            return 'provider_key_not_ok'
        }
    } catch (error) {
        logger.error('Provider key gate failed open', {
            definitionId: definition.id,
            providerKeyId,
            error: error instanceof Error ? error.message : String(error),
        })
    }

    return null
}