:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import {
Attributes,
Context,
HrTime,
Span,
SpanKind,
SpanStatusCode,
TraceFlags,
Tracer,
context,
isSpanContextValid,
trace,
} from '@opentelemetry/api'
import { Counter, Histogram, Summary, exponentialBuckets } from 'prom-client'
import { defaultConfig } from '~/common/config/config'
import { timeoutGuard } from '~/common/utils/db/utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
const instrumentedFnSummary = new Summary({
name: 'instrumented_fn_duration_ms',
help: 'Duration of instrumented functions',
labelNames: ['metricName', 'tag'],
percentiles: [0.5, 0.9, 0.95, 0.99],
})
const instrumentedFunctionDuration = new Histogram({
name: 'instrumented_function_duration_seconds',
help: 'Processing time and success status of internal functions',
labelNames: ['function', 'success'],
// We need to cover a pretty wide range, so buckets are set pretty coarse for now
// and cover 25ms -> 102seconds. We can revisit them later on.
buckets: exponentialBuckets(0.025, 4, 7),
})
const instrumentedFunctionTimeout = new Counter({
name: 'instrumented_function_timeout_total',
help: 'Number of times an instrumented function exceeded its timeout threshold',
labelNames: ['function'],
})
const logTime = (startTime: number, statsKey: string, error?: any): void => {
logger.info('⏱️', `${statsKey} took ${Math.round(performance.now() - startTime)}ms`, {
error,
statsKey,
type: 'instrumented_function_time_log',
})
}
function getHighResTimestamp(): HrTime {
// performance.timeOrigin: absolute start time of the process
// performance.now(): high-res relative time since process start
const epochMillis = performance.timeOrigin + performance.now()
const seconds = Math.floor(epochMillis / 1000)
const nanos = Math.round((epochMillis % 1000) * 1_000_000)
return [seconds, nanos]
}
// The sampler is parent-based, so a child of an unsampled span is never exported.
function hasUnsampledParent(ctx: Context): boolean {
const parent = trace.getSpanContext(ctx)
return parent !== undefined && isSpanContextValid(parent) && (parent.traceFlags & TraceFlags.SAMPLED) === 0
}
/**
* Wraps a function in an OpenTelemetry tracing span.
*/
export function withTracingSpan<T>(
tracer: Tracer | string,
name: string,
attrs: Attributes,
fn: () => Promise<T>,
parentContext?: Context
): Promise<T> {
const ctx = parentContext ?? context.active()
if (hasUnsampledParent(ctx)) {
return parentContext ? context.with(parentContext, fn) : fn()
}
const _tracer = typeof tracer === 'string' ? trace.getTracer(tracer) : tracer
const startHrTime = getHighResTimestamp()
return _tracer.startActiveSpan(
name,
{ kind: SpanKind.CLIENT, attributes: attrs, startTime: startHrTime },
ctx,
async (span) => {
try {
const out = await fn()
span.setStatus({ code: SpanStatusCode.OK })
return out
} catch (e: any) {
span.recordException(e)
span.setStatus({ code: SpanStatusCode.ERROR, message: e?.message })
throw e
} finally {
span.end(getHighResTimestamp())
}
}
)
}
async function withSummary<T>(name: string, tag: string | undefined, fn: () => Promise<T>): Promise<T> {
const stopTimer = instrumentedFnSummary.labels({ metricName: name, tag }).startTimer()
try {
return await fn()
} finally {
stopTimer()
}
}
/**
* Wraps a function in an OpenTelemetry tracing span and logs the execution time as a summary metric.
*/
export function withSpan<T>(
tracer: Tracer | string,
name: string,
attrs: Attributes,
fn: () => Promise<T>,
parentContext?: Context
): Promise<T> {
return withSummary(name, attrs.tag ? String(attrs.tag) : undefined, () =>
withTracingSpan(tracer, name, attrs, fn, parentContext)
)
}
export interface DetachedSpan {
span: Span
parentContext: Context
}
/** The span is not made active: work in another async context nests under it through `parentContext`. */
export function startDetachedSpan(name: string, attrs: Attributes, parent?: Context): DetachedSpan | null {
if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
return null
}
const parentContext = parent ?? context.active()
const span = trace
.getTracer('instrumented_function')
.startSpan(name, { kind: SpanKind.CLIENT, attributes: attrs, startTime: getHighResTimestamp() }, parentContext)
return { span, parentContext: trace.setSpan(parentContext, span) }
}
export function runInContext<T>(ctx: Context | undefined, fn: () => Promise<T>): Promise<T> {
return ctx ? context.with(ctx, fn) : fn()
}
interface FunctionInstrumentationOptions {
key: string
timeoutMs?: number
timeoutMessage?: string
getLoggingContext?: () => Record<string, any>
logExecutionTime?: boolean
sendException?: boolean
measureTime?: boolean
/** Stamped as the `tag` attribute on the span and the summary metric, to tell apart callers sharing a key. */
tag?: string
/** Attributes set on the tracing span only. They never reach the Prometheus metrics. */
attributes?: Attributes
/** Parent of the span instead of the active context, for work run by a shared loop outside its batch's async context. */
parentContext?: Context
/** False keeps the timeout guard and both duration metrics but emits no span. */
span?: boolean
/** False skips the `instrumented_fn_duration_ms` summary, for callers that record their own duration. */
summary?: boolean
}
/**
* True when a span started under `ctx` (default: the active context) would be exported. With
* parent-based sampling the parent decides, so callers can skip building attributes for the ~99%
* of spans a low sample rate discards.
*/
export function isTracingActive(ctx?: Context): boolean {
if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
return false
}
return trace.getSpan(ctx ?? context.active())?.isRecording() ?? false
}
/** Set attributes on the active span. No-op when tracing is disabled or the span is not sampled. */
export function setSpanAttributes(attrs: Attributes): void {
if (defaultConfig.DISABLE_OPENTELEMETRY_TRACING) {
return
}
const span = trace.getActiveSpan()
if (span?.isRecording()) {
span.setAttributes(attrs)
}
}
/**
* Wraps a function in a timeout guard and a prometheus metric.
*/
export async function instrumentFn<T>(
options: string | FunctionInstrumentationOptions,
func: () => Promise<T>
): Promise<T> {
const key = typeof options === 'string' ? options : options.key
const timeoutMessage =
(typeof options === 'string' ? undefined : options.timeoutMessage) ?? `Timeout warning for '${key}'!`
const getLoggingContext = (typeof options === 'string' ? undefined : options.getLoggingContext) ?? undefined
const timeout = (typeof options === 'string' ? undefined : options.timeoutMs) ?? defaultConfig.TASK_TIMEOUT * 1000
const sendException = (typeof options === 'string' ? undefined : options.sendException) ?? true
const logExecutionTime = (typeof options === 'string' ? undefined : options.logExecutionTime) ?? false
const measureTime = (typeof options === 'string' ? undefined : options.measureTime) ?? true
const tag = typeof options === 'string' ? undefined : options.tag
const attributes = (typeof options === 'string' ? undefined : options.attributes) ?? {}
const parentContext = typeof options === 'string' ? undefined : options.parentContext
const span = (typeof options === 'string' ? undefined : options.span) ?? true
const summary = (typeof options === 'string' ? undefined : options.summary) ?? true
const t = timeoutGuard(timeoutMessage, getLoggingContext, timeout, sendException, () => {
instrumentedFunctionTimeout.labels({ function: key }).inc()
})
const startTime = performance.now()
const end = measureTime ? instrumentedFunctionDuration.startTimer({ function: key }) : undefined
const funcReportingErrorsInSpan = sendException
? async (): Promise<T> => {
try {
return await func()
} catch (error) {
captureException(error)
throw error
}
}
: func
try {
// Skip expensive span creation when tracing is disabled
const tracingEnabled = !defaultConfig.DISABLE_OPENTELEMETRY_TRACING
const run =
tracingEnabled && span
? () =>
withTracingSpan(
'instrumented_function',
key,
tag ? { ...attributes, tag } : attributes,
funcReportingErrorsInSpan,
parentContext
)
: funcReportingErrorsInSpan
const result = tracingEnabled && summary ? await withSummary(key, tag, run) : await run()
end?.({ success: 'true' })
if (logExecutionTime) {
logTime(startTime, key)
}
return result
} catch (error) {
end?.({ success: 'false' })
logger.info('🔔', error)
if (logExecutionTime) {
logTime(startTime, key, error)
}
throw error
} finally {
clearTimeout(t)
}
}
/**
* Decorator that can be applied to class methods or standalone functions to add tracing and instrumentation.
*
* @param options - Either a string key or FunctionInstrumentationOptions object
* @returns A decorator function that wraps the original method/function
*
* @example
* // For class methods:
* class MyService {
* @instrumented('my-service-method')
* async myMethod() {
* // method implementation
* }
* }
*
* // For standalone functions:
* const myFunction = instrumented('my-function')(async () => {
* // function implementation
* })
*/
export function instrumented(options: string | FunctionInstrumentationOptions) {
return function <T extends (...args: any[]) => Promise<any>>(
target: any,
propertyKey?: string | symbol,
descriptor?: TypedPropertyDescriptor<T>
): any {
// If used as a method decorator (class method)
if (descriptor && propertyKey) {
const originalMethod = descriptor.value
if (!originalMethod) {
throw new Error('Cannot apply instrumented decorator to non-function property')
}
descriptor.value = async function (this: any, ...args: any[]): Promise<any> {
return instrumentFn(options, () => originalMethod.apply(this, args))
} as T
return descriptor
}
// If used as a function decorator (standalone function)
if (typeof target === 'function') {
return async function (this: any, ...args: any[]): Promise<any> {
return instrumentFn(options, () => target.apply(this, args))
}
}
throw new Error('instrumented decorator can only be applied to functions or methods')
}
}