Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

21.7 KB
Raw Blame
import { DateTime } from 'luxon'
import { Counter, Histogram } from 'prom-client'

import { HogFunctionManagerService } from '~/cdp/services/managers/hog-function-manager.service'
import { HogFunctionMonitoringService } from '~/cdp/services/monitoring/hog-function-monitoring.service'
import { HogFunctionType, LogEntry } from '~/cdp/types'
import { sanitizeLogMessage } from '~/cdp/utils'
import { yieldEventLoopIfNeeded } from '~/common/utils/event-loop-yield'
import { logger } from '~/common/utils/logger'
import { UUIDT } from '~/common/utils/utils'

import { decodeLogAttributeValue, encodeLogAttributeValue } from '../attribute-value'
import type { LogRecord } from '../log-record-avro'
import {
    LogTransformationGlobals,
    buildLogRecordGlobalsRecord,
    executeLogTransformation,
    refreshLogRecordGlobalsRecord,
    resolveLogTransformationInputs,
} from './hog-log-exec'

// Iterative with a visited set: no depth cap (a secret below any cap would escape
// redaction entirely) and safe against cyclic values.
function collectStringLeaves(root: unknown, out: string[]): void {
    const stack: unknown[] = [root]
    const seen = new WeakSet<object>()
    while (stack.length > 0) {
        const value = stack.pop()
        if (typeof value === 'string') {
            if (value.length > 0) {
                out.push(value)
            }
            continue
        }
        if (value && typeof value === 'object') {
            if (seen.has(value)) {
                continue
            }
            seen.add(value)
            // Push one-by-one: spreading an unbounded user-supplied array would
            // exceed V8's argument limit and throw RangeError.
            for (const item of Array.isArray(value) ? value : Object.values(value)) {
                stack.push(item)
            }
        }
    }
}

export const transformationRecordsCounter = new Counter({
    name: 'logs_ingestion_transformations_records_total',
    help: 'Per-record log transformation outcomes',
    labelNames: ['result'], // succeeded | failed | dropped | budget_skipped | watcher_disabled
})

export const transformationVmDurationHistogram = new Histogram({
    name: 'logs_ingestion_transformations_vm_duration_seconds',
    help: 'Total HogVM execution time spent on log transformations per Kafka message',
    buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2],
})

export const transformationBudgetExhaustedCounter = new Counter({
    name: 'logs_ingestion_transformations_budget_exhausted_total',
    help: 'Transformation time budget exhausted; remaining records passed through untransformed',
    labelNames: ['scope', 'team_id'], // scope: message | batch
})

export const transformationUnexpectedErrorsCounter = new Counter({
    name: 'logs_ingestion_transformations_unexpected_errors_total',
    help: 'Unexpected (non-customer-code) errors in the logs transformer. Any occurrence should alert.',
})

export const transformationInfraErrorsCounter = new Counter({
    name: 'logs_ingestion_transformations_infra_errors_total',
    help: 'Infrastructure errors while fetching functions or reading watcher state; function fetch errors pass records through untransformed.',
    labelNames: ['source'], // functions_fetch | watcher_state
})

export interface LogsTransformerConfig {
    siteUrl: string
    /** Hard per-record VM kill */
    hogTimeoutMs: number
    /** Cumulative VM time budget per Kafka message */
    messageBudgetMs: number
    /** Cumulative VM time budget per consumer batch */
    batchBudgetMs: number
    /** Max failed invocations whose print/error logs are captured, per function per message */
    maxErrorLogsPerFunctionPerMessage: number
}

/** Tracks cumulative VM time across all messages of one consumer batch. */
export class TransformationBatchBudget {
    usedMs = 0

    constructor(private readonly totalMs: number) {}

    get exhausted(): boolean {
        return this.usedMs >= this.totalMs
    }

    add(durationMs: number): void {
        this.usedMs += durationMs
    }
}

interface FunctionAggregates {
    succeeded: number
    failed: number
    dropped: number
    budgetSkipped: number
    totalDurationMs: number
    errorLogs: LogEntry[]
}

const ATTR_TRANSFORMATIONS_FAILED = '$transformations_failed'
const ATTR_TRANSFORMATIONS_SKIPPED = '$transformations_skipped'

/**
 * Runs a team's enabled log transformations over the records of one Kafka message,
 * mutating the array in place (dropped records are removed).
 *
 * Failure semantics are fail-open throughout, mirroring events ingestion and the
 * logs sampling pipeline: a failing transformation annotates the record and passes
 * it through unchanged. Budgets bound the worst-case added latency; when exhausted,
 * remaining records skip transformation and are annotated.
 *
 * Unlike the events HogTransformerService there are no per-invocation result objects:
 * at logs volume (100k+ records/s for large teams) metrics are aggregated per
 * (function, message) and print() output is only kept for failing invocations.
 */
export class LogsTransformerService {
    constructor(
        private hogFunctionManager: HogFunctionManagerService,
        private monitoring: HogFunctionMonitoringService,
        private config: LogsTransformerConfig
    ) {}

    public startBatch(): TransformationBatchBudget {
        return new TransformationBatchBudget(this.config.batchBudgetMs)
    }

    /** Cheap existence check (in-process LazyLoader cache) used to preserve the
     * no-decode passthrough for teams without transformations. */
    public async teamHasTransformations(teamId: number): Promise<boolean> {
        // Fail open: a fetch failure (cold cache + Postgres blip) must skip transformations
        // for the message, never bubble into the consumer's catch and DLQ valid records.
        try {
            const idsByTeam = await this.hogFunctionManager.getHogFunctionIdsForTeams([teamId], ['transformation_log'])
            return (idsByTeam[teamId] ?? []).length > 0
        } catch (error) {
            transformationInfraErrorsCounter.inc({ source: 'functions_fetch' })
            logger.warn('[logs-transformer] function id fetch failed — skipping transformations', {
                teamId,
                error: String(error),
            })
            return false
        }
    }

    public async flush(): Promise<void> {
        await this.monitoring.flush()
    }

    public async transformRecords(
        teamId: number,
        records: LogRecord[],
        batchBudget?: TransformationBatchBudget,
        sentAsInternalLogs = false
    ): Promise<{ recordsDropped: number; recordsDroppedByFunctionId: Map<string, number> }> {
        const recordsDroppedByFunctionId = new Map<string, number>()
        let recordsDropped = 0

        let allFunctions: HogFunctionType[]
        try {
            const functionsByTeam = await this.hogFunctionManager.getHogFunctionsForTeams(
                [teamId],
                ['transformation_log']
            )
            allFunctions = functionsByTeam[teamId] ?? []
        } catch (error) {
            // Fail open like the fetch in teamHasTransformations: pass records through
            // untransformed rather than DLQ the message.
            transformationInfraErrorsCounter.inc({ source: 'functions_fetch' })
            logger.warn('[logs-transformer] function fetch failed — records passed through untransformed', {
                teamId,
                error: String(error),
            })
            return { recordsDropped, recordsDroppedByFunctionId }
        }
        if (allFunctions.length === 0 || records.length === 0) {
            return { recordsDropped, recordsDroppedByFunctionId }
        }

        // On a sampled message the watcher reads state (to skip disabled functions) and, at the
        // end, receives the aggregate cost. A sample rate of 0 means no Redis traffic at all.
        const functions = allFunctions
        if (functions.length === 0) {
            return { recordsDropped, recordsDroppedByFunctionId }
        }

        const project = {
            id: teamId,
            name: '',
            url: `${this.config.siteUrl}/project/${teamId}`,
        }
        const instanceId = new UUIDT().toString()
        const aggregates = new Map<string, FunctionAggregates>()
        const inputsCache = new Map<string, Record<string, unknown> | null>()
        const sensitiveValuesCache = new Map<string, string[]>()
        let messageVmMs = 0
        let budgetExhaustedScope: 'message' | 'batch' | null = null

        const kept: LogRecord[] = []

        for (let i = 0; i < records.length; i++) {
            const record = records[i]

            if (budgetExhaustedScope === null) {
                if (batchBudget?.exhausted) {
                    budgetExhaustedScope = 'batch'
                } else if (messageVmMs >= this.config.messageBudgetMs) {
                    budgetExhaustedScope = 'message'
                }
                if (budgetExhaustedScope !== null) {
                    transformationBudgetExhaustedCounter.inc({
                        scope: budgetExhaustedScope,
                        team_id: String(teamId),
                    })
                }
            }

            if (budgetExhaustedScope !== null) {
                record.attributes = record.attributes ?? {}
                // JSON-encoded like every attribute value, so the ClickHouse sink surfaces it
                record.attributes[ATTR_TRANSFORMATIONS_SKIPPED] = encodeLogAttributeValue('budget')
                transformationRecordsCounter.inc({ result: 'budget_skipped' })
                for (const fn of functions) {
                    this.getAggregates(aggregates, fn.id).budgetSkipped++
                }
                kept.push(record)
                continue
            }

            const dropped = await yieldEventLoopIfNeeded('logs-transformer', () => {
                return this.transformSingleRecord(record, functions, project, {
                    teamId,
                    instanceId,
                    aggregates,
                    inputsCache,
                    sensitiveValuesCache,
                    recordsDroppedByFunctionId,
                    addVmMs: (ms) => {
                        messageVmMs += ms
                        batchBudget?.add(ms)
                    },
                })
            })

            if (dropped) {
                recordsDropped++
            } else {
                kept.push(record)
            }
        }

        // Replace contents in place so callers holding the array reference see the result.
        // One-by-one push: spreading an unbounded record array would exceed V8's argument
        // limit and throw RangeError, escaping the per-record fail-open handling.
        records.length = 0
        for (const record of kept) {
            records.push(record)
        }

        transformationVmDurationHistogram.observe(messageVmMs / 1000)
        this.queueAggregates(teamId, aggregates, sentAsInternalLogs)
        return { recordsDropped, recordsDroppedByFunctionId }
    }

    /** Runs every function in execution order against one record. Returns true if dropped. */
    private transformSingleRecord(
        record: LogRecord,
        functions: HogFunctionType[],
        project: LogTransformationGlobals['project'],
        ctx: {
            teamId: number
            instanceId: string
            aggregates: Map<string, FunctionAggregates>
            inputsCache: Map<string, Record<string, unknown> | null>
            sensitiveValuesCache: Map<string, string[]>
            recordsDroppedByFunctionId: Map<string, number>
            addVmMs: (ms: number) => void
        }
    ): boolean {
        // Built once per record, not per function: the decoded maps change only when a
        // function transforms the record, so globals are refreshed from the record
        // between functions instead of rebuilt.
        const globals: LogTransformationGlobals = {
            project,
            record: buildLogRecordGlobalsRecord(record),
            inputs: {},
        }
        // Refreshed only after a function actually wrote to the record (a mutation or a
        // failure annotation): most functions leave most records untouched.
        let recordDirty = false
        for (const fn of functions) {
            const agg = this.getAggregates(ctx.aggregates, fn.id)

            if (recordDirty) {
                refreshLogRecordGlobalsRecord(globals.record, record)
                recordDirty = false
            }

            // Input templates are customer-owned bytecode — a failing template is an
            // invocation failure (fail open, annotate, surface in function logs), not a
            // transformer bug worth alerting on.
            try {
                const { inputs, vmMs } = this.resolveInputs(fn, globals, ctx.inputsCache)
                globals.inputs = inputs
                ctx.addVmMs(vmMs)
                agg.totalDurationMs += vmMs
            } catch (error) {
                agg.failed++
                transformationRecordsCounter.inc({ result: 'failed' })
                this.annotateFailure(record, fn)
                recordDirty = true
                this.captureErrorLogs(
                    fn,
                    ctx.teamId,
                    ctx.instanceId,
                    agg,
                    [],
                    String(error),
                    this.getSensitiveValues(fn, ctx.sensitiveValuesCache)
                )
                continue
            }

            let outcome
            try {
                outcome = executeLogTransformation(fn.bytecode, record, globals, {
                    timeoutMs: this.config.hogTimeoutMs,
                    sensitiveValues: this.getSensitiveValues(fn, ctx.sensitiveValuesCache),
                })
            } catch (error) {
                // Errors from customer code are contained inside executeLogTransformation;
                // reaching here means a transformer bug.
                transformationUnexpectedErrorsCounter.inc()
                logger.error('⚠️', '[logs-transformer] unexpected error', {
                    teamId: ctx.teamId,
                    functionId: fn.id,
                    error: String(error),
                })
                this.annotateFailure(record, fn)
                recordDirty = true
                agg.failed++
                transformationRecordsCounter.inc({ result: 'failed' })
                continue
            }

            ctx.addVmMs(outcome.durationMs)
            agg.totalDurationMs += outcome.durationMs

            if (outcome.status === 'failed') {
                agg.failed++
                transformationRecordsCounter.inc({ result: 'failed' })
                this.annotateFailure(record, fn)
                recordDirty = true
                this.captureErrorLogs(
                    fn,
                    ctx.teamId,
                    ctx.instanceId,
                    agg,
                    outcome.logs,
                    outcome.error,
                    this.getSensitiveValues(fn, ctx.sensitiveValuesCache)
                )
                continue
            }

            if (outcome.status === 'dropped') {
                agg.dropped++
                transformationRecordsCounter.inc({ result: 'dropped' })
                ctx.recordsDroppedByFunctionId.set(fn.id, (ctx.recordsDroppedByFunctionId.get(fn.id) ?? 0) + 1)
                return true
            }

            agg.succeeded++
            transformationRecordsCounter.inc({ result: 'succeeded' })
            recordDirty = true
        }

        return false
    }

    /**
     * Resolves function inputs once per (function, message) and caches them — unless any
     * input template references the `record` global, in which case resolution happens per
     * record (cache stores null to remember that decision).
     */
    private resolveInputs(
        fn: HogFunctionType,
        globals: LogTransformationGlobals,
        cache: Map<string, Record<string, unknown> | null>
    ): { inputs: Record<string, unknown>; vmMs: number } {
        const cached = cache.get(fn.id)
        if (cached) {
            return { inputs: cached, vmMs: 0 }
        }

        if (cached === undefined) {
            // First sight of this function in this message: decide cacheability with a
            // cheap scan over all templates (encrypted included). False positives only
            // cost per-record resolution.
            const referencesRecord = JSON.stringify({ ...fn.inputs, ...fn.encrypted_inputs }).includes('record')
            if (!referencesRecord) {
                const resolved = this.resolveInputsUncached(fn, globals)
                cache.set(fn.id, resolved.inputs)
                return { inputs: resolved.inputs, vmMs: resolved.durationMs }
            }
            cache.set(fn.id, null)
        }

        const resolved = this.resolveInputsUncached(fn, globals)
        return { inputs: resolved.inputs, vmMs: resolved.durationMs }
    }

    private resolveInputsUncached(
        fn: HogFunctionType,
        globals: LogTransformationGlobals
    ): { inputs: Record<string, unknown>; durationMs: number } {
        return resolveLogTransformationInputs(fn, globals, this.config.hogTimeoutMs)
    }

    private getSensitiveValues(fn: HogFunctionType, cache: Map<string, string[]>): string[] {
        let values = cache.get(fn.id)
        if (!values) {
            // Secret inputs are schema-validated as any JSON value, so redaction must
            // cover string leaves nested in dicts/arrays, not just top-level strings.
            values = []
            for (const input of Object.values(fn.encrypted_inputs ?? {})) {
                collectStringLeaves(input?.value, values)
            }
            cache.set(fn.id, values)
        }
        return values
    }

    private annotateFailure(record: LogRecord, fn: HogFunctionType): void {
        record.attributes = record.attributes ?? {}
        const rawExisting = record.attributes[ATTR_TRANSFORMATIONS_FAILED]
        const existing = rawExisting ? decodeLogAttributeValue(rawExisting) : undefined
        const identifier = `${fn.name} (${fn.id})`
        record.attributes[ATTR_TRANSFORMATIONS_FAILED] = encodeLogAttributeValue(
            existing ? `${existing}, ${identifier}` : identifier
        )
    }

    private captureErrorLogs(
        fn: HogFunctionType,
        teamId: number,
        instanceId: string,
        agg: FunctionAggregates,
        printLogs: string[],
        error: string,
        sensitiveValues: string[]
    ): void {
        const capturedFailures = agg.errorLogs.filter((log) => log.level === 'error').length
        if (capturedFailures >= this.config.maxErrorLogsPerFunctionPerMessage) {
            return
        }

        const base = {
            team_id: teamId,
            log_source: 'hog_function' as const,
            log_source_id: fn.id,
            instance_id: instanceId,
        }
        for (const message of printLogs) {
            agg.errorLogs.push({ ...base, level: 'info', message, timestamp: DateTime.now() })
        }
        // The thrown error string is customer-influenced like print() output —
        // redact encrypted input values before it reaches persisted function logs.
        agg.errorLogs.push({
            ...base,
            level: 'error',
            message: sanitizeLogMessage([error], sensitiveValues),
            timestamp: DateTime.now(),
        })
    }

    private getAggregates(aggregates: Map<string, FunctionAggregates>, functionId: string): FunctionAggregates {
        let agg = aggregates.get(functionId)
        if (!agg) {
            agg = { succeeded: 0, failed: 0, dropped: 0, budgetSkipped: 0, totalDurationMs: 0, errorLogs: [] }
            aggregates.set(functionId, agg)
        }
        return agg
    }

    private queueAggregates(
        teamId: number,
        aggregates: Map<string, FunctionAggregates>,
        sentAsInternalLogs: boolean
    ): void {
        for (const [functionId, agg] of aggregates) {
            if (agg.succeeded > 0) {
                this.monitoring.queueAppMetric(
                    {
                        team_id: teamId,
                        app_source_id: functionId,
                        metric_kind: 'success',
                        metric_name: 'succeeded',
                        count: agg.succeeded,
                    },
                    'hog_function'
                )
            }
            if (agg.failed > 0) {
                this.monitoring.queueAppMetric(
                    {
                        team_id: teamId,
                        app_source_id: functionId,
                        metric_kind: 'failure',
                        metric_name: 'failed',
                        count: agg.failed,
                    },
                    'hog_function'
                )
            }
            if (agg.dropped > 0) {
                this.monitoring.queueAppMetric(
                    {
                        team_id: teamId,
                        app_source_id: functionId,
                        metric_kind: 'other',
                        metric_name: 'dropped',
                        count: agg.dropped,
                    },
                    'hog_function'
                )
            }
            if (agg.budgetSkipped > 0) {
                this.monitoring.queueAppMetric(
                    {
                        team_id: teamId,
                        app_source_id: functionId,
                        metric_kind: 'other',
                        metric_name: 'budget_skipped',
                        count: agg.budgetSkipped,
                    },
                    'hog_function'
                )
            }
            if (agg.errorLogs.length > 0) {
                this.monitoring.queueLogs(agg.errorLogs, 'hog_function', { aboutInternalLogs: sentAsInternalLogs })
            }
        }
    }
}