Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.2 KB
Raw Blame
import {
    Attributes,
    Context,
    HrTime,
    Span,
    SpanKind,
    SpanStatusCode,
    TraceFlags,
    Tracer,
    context,
    isSpanContextValid,
    trace,
} from '@opentelemetry/api'
import { Counter, Histogram, Summary, exponentialBuckets } from 'prom-client'

import { defaultConfig } from '~/common/config/config'
import { timeoutGuard } from '~/common/utils/db/utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'

const instrumentedFnSummary = new Summary({
    name: 'instrumented_fn_duration_ms',
    help: 'Duration of instrumented functions',
    labelNames: ['metricName', 'tag'],
    percentiles: [0.5, 0.9, 0.95, 0.99],
})

const instrumentedFunctionDuration = new Histogram({
    name: 'instrumented_function_duration_seconds',
    help: 'Processing time and success status of internal functions',
    labelNames: ['function', 'success'],
    // We need to cover a pretty wide range, so buckets are set pretty coarse for now
    // and cover 25ms -> 102seconds. We can revisit them later on.
    buckets: exponentialBuckets(0.025, 4, 7),
})

const instrumentedFunctionTimeout = new Counter({
    name: 'instrumented_function_timeout_total',
    help: 'Number of times an instrumented function exceeded its timeout threshold',
    labelNames: ['function'],
})

const logTime = (startTime: number, statsKey: string, error?: any): void => {
    logger.info('⏱️', `${statsKey} took ${Math.round(performance.now() - startTime)}ms`, {
        error,
        statsKey,
        type: 'instrumented_function_time_log',
    })
}

function getHighResTimestamp(): HrTime {
    // performance.timeOrigin: absolute start time of the process
    // performance.now(): high-res relative time since process start
    const epochMillis = performance.timeOrigin + performance.now()
    const seconds = Math.floor(epochMillis / 1000)
    const nanos = Math.round((epochMillis % 1000) * 1_000_000)
    return [seconds, nanos]
}

// The sampler is parent-based, so a child of an unsampled span is never exported.
function hasUnsampledParent(ctx: Context): boolean {
    const parent = trace.getSpanContext(ctx)
    return parent !== undefined && isSpanContextValid(parent) && (parent.traceFlags & TraceFlags.SAMPLED) === 0
}

/**
 * Wraps a function in an OpenTelemetry tracing span.
 */
export function withTracingSpan<T>(
    tracer: Tracer | string,
    name: string,
    attrs: Attributes,
    fn: () => Promise<T>,
    parentContext?: Context
): Promise<T> {
    const ctx = parentContext ?? context.active()
    if (hasUnsampledParent(ctx)) {
        return parentContext ? context.with(parentContext, fn) : fn()
    }
    const _tracer = typeof tracer === 'string' ? trace.getTracer(tracer) : tracer
    const startHrTime = getHighResTimestamp()
    return _tracer.startActiveSpan(
        name,
        { kind: SpanKind.CLIENT, attributes: attrs, startTime: startHrTime },
        ctx,
        async (span) => {
            try {
                const out = await fn()
                span.setStatus({ code: SpanStatusCode.OK })
                return out
            } catch (e: any) {
                span.recordException(e)
                span.setStatus({ code: SpanStatusCode.ERROR, message: e?.message })
                throw e
            } finally {
                span.end(getHighResTimestamp())
            }
        }
    )
}

async function withSummary<T>(name: string, tag: string | undefined, fn: () => Promise<T>): Promise<T> {
    const stopTimer = instrumentedFnSummary.labels({ metricName: name, tag }).startTimer()
    try {
        return await fn()
    } finally {
        stopTimer()
    }
}

/**
 * Wraps a function in an OpenTelemetry tracing span and logs the execution time as a summary metric.
 */
export function withSpan<T>(
    tracer: Tracer | string,
    name: string,
    attrs: Attributes,
    fn: () => Promise<T>,
    parentContext?: Context
): Promise<T> {
    return withSummary(name, attrs.tag ? String(attrs.tag) : undefined, () =>
        withTracingSpan(tracer, name, attrs, fn, parentContext)
    )
}

export interface DetachedSpan {
    span: Span
    parentContext: Context
}

/** The span is not made active: work in another async context nests under it through `parentContext`. */
export function startDetachedSpan(name: string, attrs: Attributes, parent?: Context): DetachedSpan | null {
    if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
        return null
    }
    const parentContext = parent ?? context.active()
    const span = trace
        .getTracer('instrumented_function')
        .startSpan(name, { kind: SpanKind.CLIENT, attributes: attrs, startTime: getHighResTimestamp() }, parentContext)
    return { span, parentContext: trace.setSpan(parentContext, span) }
}

export function runInContext<T>(ctx: Context | undefined, fn: () => Promise<T>): Promise<T> {
    return ctx ? context.with(ctx, fn) : fn()
}

interface FunctionInstrumentationOptions {
    key: string
    timeoutMs?: number
    timeoutMessage?: string
    getLoggingContext?: () => Record<string, any>
    logExecutionTime?: boolean
    sendException?: boolean
    measureTime?: boolean
    /** Stamped as the `tag` attribute on the span and the summary metric, to tell apart callers sharing a key. */
    tag?: string
    /** Attributes set on the tracing span only. They never reach the Prometheus metrics. */
    attributes?: Attributes
    /** Parent of the span instead of the active context, for work run by a shared loop outside its batch's async context. */
    parentContext?: Context
    /** False keeps the timeout guard and both duration metrics but emits no span. */
    span?: boolean
    /** False skips the `instrumented_fn_duration_ms` summary, for callers that record their own duration. */
    summary?: boolean
}

/**
 * True when a span started under `ctx` (default: the active context) would be exported. With
 * parent-based sampling the parent decides, so callers can skip building attributes for the ~99%
 * of spans a low sample rate discards.
 */
export function isTracingActive(ctx?: Context): boolean {
    if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
        return false
    }
    return trace.getSpan(ctx ?? context.active())?.isRecording() ?? false
}

/** Set attributes on the active span. No-op when tracing is disabled or the span is not sampled. */
export function setSpanAttributes(attrs: Attributes): void {
    if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
        return
    }
    const span = trace.getActiveSpan()
    if (span?.isRecording()) {
        span.setAttributes(attrs)
    }
}

/**
 * Wraps a function in a timeout guard and a prometheus metric.
 */

export async function instrumentFn<T>(
    options: string | FunctionInstrumentationOptions,
    func: () => Promise<T>
): Promise<T> {
    const key = typeof options === 'string' ? options : options.key
    const timeoutMessage =
        (typeof options === 'string' ? undefined : options.timeoutMessage) ?? `Timeout warning for '${key}'!`
    const getLoggingContext = (typeof options === 'string' ? undefined : options.getLoggingContext) ?? undefined
    const timeout = (typeof options === 'string' ? undefined : options.timeoutMs) ?? defaultConfig.TASK_TIMEOUT * 1000
    const sendException = (typeof options === 'string' ? undefined : options.sendException) ?? true
    const logExecutionTime = (typeof options === 'string' ? undefined : options.logExecutionTime) ?? false
    const measureTime = (typeof options === 'string' ? undefined : options.measureTime) ?? true
    const tag = typeof options === 'string' ? undefined : options.tag
    const attributes = (typeof options === 'string' ? undefined : options.attributes) ?? {}
    const parentContext = typeof options === 'string' ? undefined : options.parentContext
    const span = (typeof options === 'string' ? undefined : options.span) ?? true
    const summary = (typeof options === 'string' ? undefined : options.summary) ?? true

    const t = timeoutGuard(timeoutMessage, getLoggingContext, timeout, sendException, () => {
        instrumentedFunctionTimeout.labels({ function: key }).inc()
    })
    const startTime = performance.now()
    const end = measureTime ? instrumentedFunctionDuration.startTimer({ function: key }) : undefined
    const funcReportingErrorsInSpan = sendException
        ? async (): Promise<T> => {
              try {
                  return await func()
              } catch (error) {
                  captureException(error)
                  throw error
              }
          }
        : func

    try {
        // Skip expensive span creation when tracing is disabled
        const tracingEnabled = !defaultConfig.DISABLE_OPENTELEMETRY_TRACING
        const run =
            tracingEnabled && span
                ? () =>
                      withTracingSpan(
                          'instrumented_function',
                          key,
                          tag ? { ...attributes, tag } : attributes,
                          funcReportingErrorsInSpan,
                          parentContext
                      )
                : funcReportingErrorsInSpan
        const result = tracingEnabled && summary ? await withSummary(key, tag, run) : await run()
        end?.({ success: 'true' })
        if (logExecutionTime) {
            logTime(startTime, key)
        }
        return result
    } catch (error) {
        end?.({ success: 'false' })
        logger.info('🔔', error)
        if (logExecutionTime) {
            logTime(startTime, key, error)
        }
        throw error
    } finally {
        clearTimeout(t)
    }
}

/**
 * Decorator that can be applied to class methods or standalone functions to add tracing and instrumentation.
 *
 * @param options - Either a string key or FunctionInstrumentationOptions object
 * @returns A decorator function that wraps the original method/function
 *
 * @example
 * // For class methods:
 * class MyService {
 *   @instrumented('my-service-method')
 *   async myMethod() {
 *     // method implementation
 *   }
 * }
 *
 * // For standalone functions:
 * const myFunction = instrumented('my-function')(async () => {
 *   // function implementation
 * })
 */
export function instrumented(options: string | FunctionInstrumentationOptions) {
    return function <T extends (...args: any[]) => Promise<any>>(
        target: any,
        propertyKey?: string | symbol,
        descriptor?: TypedPropertyDescriptor<T>
    ): any {
        // If used as a method decorator (class method)
        if (descriptor && propertyKey) {
            const originalMethod = descriptor.value
            if (!originalMethod) {
                throw new Error('Cannot apply instrumented decorator to non-function property')
            }

            descriptor.value = async function (this: any, ...args: any[]): Promise<any> {
                return instrumentFn(options, () => originalMethod.apply(this, args))
            } as T

            return descriptor
        }

        // If used as a function decorator (standalone function)
        if (typeof target === 'function') {
            return async function (this: any, ...args: any[]): Promise<any> {
                return instrumentFn(options, () => target.apply(this, args))
            }
        }

        throw new Error('instrumented decorator can only be applied to functions or methods')
    }
}