:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(ingestion): cut per-step instrumentation cost on the event hot path (#123908)
Paweł Ledwońffd0fb0a7a8e
diff --git a/nodejs/src/common/tracing/tracing-utils.test.ts b/nodejs/src/common/tracing/tracing-utils.test.tsindex 665490b6a891..35a3bb676c2b 100644--- a/nodejs/src/common/tracing/tracing-utils.test.ts+++ b/nodejs/src/common/tracing/tracing-utils.test.ts@@ -1,4 +1,4 @@-import { context, propagation, trace } from '@opentelemetry/api'+import { ROOT_CONTEXT, TraceFlags, context, propagation, trace } from '@opentelemetry/api'import { InMemorySpanExporter, NodeTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node'import { register } from 'prom-client'@@ -51,6 +51,23 @@ describe('instrumentFn', () => {expect(byName.step.parentSpanContext?.spanId).toBe(byName.batch.spanContext().spanId)})+ it('runs work under an unsampled parentContext inside that context, so nested spans stay unsampled', async () => {+ const unsampledParent = trace.setSpanContext(ROOT_CONTEXT, {+ traceId: '0af7651916cd43dd8448eb211c80319c',+ spanId: 'b7ad6b7169203331',+ traceFlags: TraceFlags.NONE,+ })++ await instrumentFn({ key: 'step', parentContext: unsampledParent }, () =>+ trace.getTracer('test').startActiveSpan('nested', (nested) => {+ nested.end()+ return Promise.resolve()+ })+ )++ expect(exporter.getFinishedSpans()).toEqual([])+ })+it('captures a failure while its span is active, so the error carries its trace id', async () => {let activeSpanIdAtCapture: string | undefinedjest.spyOn(posthogUtils, 'captureException').mockImplementationOnce(() => {diff --git a/nodejs/src/common/tracing/tracing-utils.ts b/nodejs/src/common/tracing/tracing-utils.tsindex ac8a69ec0587..4fa8701723d9 100644--- a/nodejs/src/common/tracing/tracing-utils.ts+++ b/nodejs/src/common/tracing/tracing-utils.ts@@ -1,4 +1,16 @@-import { Attributes, Context, HrTime, Span, SpanKind, SpanStatusCode, Tracer, context, trace } from '@opentelemetry/api'+import {+ Attributes,+ Context,+ HrTime,+ Span,+ SpanKind,+ SpanStatusCode,+ TraceFlags,+ Tracer,+ context,+ isSpanContextValid,+ trace,+} from '@opentelemetry/api'import { Counter, Histogram, Summary, exponentialBuckets } from 'prom-client'import { defaultConfig } from '~/common/config/config'@@ -44,6 +56,13 @@ function getHighResTimestamp(): HrTime {const nanos = Math.round((epochMillis % 1000) * 1_000_000)return [seconds, nanos]}++// The sampler is parent-based, so a child of an unsampled span is never exported.+function hasUnsampledParent(ctx: Context): boolean {+ const parent = trace.getSpanContext(ctx)+ return parent !== undefined && isSpanContextValid(parent) && (parent.traceFlags & TraceFlags.SAMPLED) === 0+}+/*** Wraps a function in an OpenTelemetry tracing span.*/@@ -54,12 +73,16 @@ export function withTracingSpan<T>(fn: () => Promise<T>,parentContext?: Context): Promise<T> {+ const ctx = parentContext ?? context.active()+ if (hasUnsampledParent(ctx)) {+ return parentContext ? context.with(parentContext, fn) : fn()+ }const _tracer = typeof tracer === 'string' ? trace.getTracer(tracer) : tracerconst startHrTime = getHighResTimestamp()return _tracer.startActiveSpan(name,{ kind: SpanKind.CLIENT, attributes: attrs, startTime: startHrTime },- parentContext ?? context.active(),+ ctx,async (span) => {try {const out = await fn()@@ -137,6 +160,8 @@ interface FunctionInstrumentationOptions {parentContext?: Context/** False keeps the timeout guard and both duration metrics but emits no span. */span?: boolean+ /** False skips the `instrumented_fn_duration_ms` summary, for callers that record their own duration. */+ summary?: boolean}/**@@ -182,6 +207,7 @@ export async function instrumentFn<T>(const attributes = (typeof options === 'string' ? undefined : options.attributes) ?? {}const parentContext = typeof options === 'string' ? undefined : options.parentContextconst span = (typeof options === 'string' ? undefined : options.span) ?? true+ const summary = (typeof options === 'string' ? undefined : options.summary) ?? trueconst t = timeoutGuard(timeoutMessage, getLoggingContext, timeout, sendException, () => {instrumentedFunctionTimeout.labels({ function: key }).inc()@@ -201,17 +227,19 @@ export async function instrumentFn<T>(try {// Skip expensive span creation when tracing is disabled- const result = defaultConfig.DISABLE_OPENTELEMETRY_TRACING- ? await funcReportingErrorsInSpan()- : span- ? await withSpan(- 'instrumented_function',- key,- tag ? { ...attributes, tag } : attributes,- funcReportingErrorsInSpan,- parentContext- )- : await withSummary(key, tag, funcReportingErrorsInSpan)+ const tracingEnabled = !defaultConfig.DISABLE_OPENTELEMETRY_TRACING+ const run =+ tracingEnabled && span+ ? () =>+ withTracingSpan(+ 'instrumented_function',+ key,+ tag ? { ...attributes, tag } : attributes,+ funcReportingErrorsInSpan,+ parentContext+ )+ : funcReportingErrorsInSpan+ const result = tracingEnabled && summary ? await withSummary(key, tag, run) : await run()end?.({ success: 'true' })if (logExecutionTime) {logTime(startTime, key)diff --git a/nodejs/src/common/utils/lazy-loader.ts b/nodejs/src/common/utils/lazy-loader.tsindex 880e178132fa..0e5d72431b01 100644--- a/nodejs/src/common/utils/lazy-loader.ts+++ b/nodejs/src/common/utils/lazy-loader.ts@@ -233,56 +233,57 @@ export class LazyLoader<T> {* If the value is older than the refreshAge, it is loaded from the database.*/private async loadViaCache(keys: string[], options?: LoadOptions): Promise<Record<string, T | null>> {- return await instrumentFn({ key: `lazyLoader.loadViaCache`, tag: this.options.name, span: false }, async () => {- // No prototype, for the same reason as the cache: keys are caller-supplied, and this- // object is handed back to callers who may iterate or spread it.- const results: Record<string, T | null> = Object.create(null)- const keysToLoad = new Set<string>()-- for (const key of keys) {- const cached = this.cache[key]-- if (cached !== undefined) {- results[key] = cached.value- cached.lastUsed = Date.now()-- const cacheUntil = cached.cacheUntil- const backgroundRefreshAfter = cached.backgroundRefreshAfter-- if (Date.now() > cacheUntil) {- keysToLoad.add(key)- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()- continue- }+ // No prototype, for the same reason as the cache: keys are caller-supplied, and this+ // object is handed back to callers who may iterate or spread it.+ const results: Record<string, T | null> = Object.create(null)+ const keysToLoad = new Set<string>()- if (backgroundRefreshAfter && Date.now() > backgroundRefreshAfter) {- void this.load([key]).catch((err) => {- logger.warn(`[LazyLoader:${this.options.name}] Background refresh failed`, {- key,- error: String(err),- })- })- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit_background' }).inc()- continue- }- } else {+ for (const key of keys) {+ const cached = this.cache[key]++ if (cached !== undefined) {+ results[key] = cached.value+ cached.lastUsed = Date.now()++ const cacheUntil = cached.cacheUntil+ const backgroundRefreshAfter = cached.backgroundRefreshAfter++ if (Date.now() > cacheUntil) {keysToLoad.add(key)lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()continue}- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()+ if (backgroundRefreshAfter && Date.now() > backgroundRefreshAfter) {+ void this.load([key]).catch((err) => {+ logger.warn(`[LazyLoader:${this.options.name}] Background refresh failed`, {+ key,+ error: String(err),+ })+ })+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit_background' }).inc()+ continue+ }+ } else {+ keysToLoad.add(key)+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()+ continue}- if (keysToLoad.size === 0) {- lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()- return results- }+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()+ }++ if (keysToLoad.size === 0) {+ lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()+ return results+ }- lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()+ lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()- // Only a miss gets a span: with one span per lookup, hits outnumber every other span in a trace.- return await withTracingSpan(+ // Only a miss is instrumented: hits would outnumber every other span in a trace, and on a hit the+ // instrumentation costs more than the lookup.+ return await instrumentFn({ key: `lazyLoader.loadViaCache`, tag: this.options.name, span: false }, () =>+ withTracingSpan('instrumented_function','lazyLoader.loadViaCache',{ tag: this.options.name },@@ -301,7 +302,7 @@ export class LazyLoader<T> {return results})- })+ )}private spanAttributes(keys: string[], keysToLoad: Set<string>, outcome: LazyLoaderSpanOutcome): Attributes {diff --git a/nodejs/src/ingestion/framework/base-chunk-pipeline.ts b/nodejs/src/ingestion/framework/base-chunk-pipeline.tsindex 94960094dfed..93b184bc70f4 100644--- a/nodejs/src/ingestion/framework/base-chunk-pipeline.ts+++ b/nodejs/src/ingestion/framework/base-chunk-pipeline.ts@@ -82,7 +82,7 @@ export async function applyChunkStepToResults<TIn, TOut, C, RPrev extends stringconst end = pipelineStepDurationHistogram.startTimer({ step_name: stepName, step_type: 'chunk' })try {stepResults = await instrumentFn(- { key: stepName, sendException: false, measureTime: false, attributes, parentContext },+ { key: stepName, sendException: false, measureTime: false, summary: false, attributes, parentContext },() => step(successfulValues))end({ result: 'chunk' })diff --git a/nodejs/src/ingestion/framework/step-pipeline.ts b/nodejs/src/ingestion/framework/step-pipeline.tsindex 7a5c9f529fa6..fe204d6f5a64 100644--- a/nodejs/src/ingestion/framework/step-pipeline.ts+++ b/nodejs/src/ingestion/framework/step-pipeline.ts@@ -57,6 +57,7 @@ export class StepPipeline<TInput, TIntermediate, TOutput, C, RPrev extends strinkey: this.stepName,sendException: false,measureTime: false,+ summary: false,attributes: isTracingActive(traceContext) ? eventSpanAttributes(previousResult.value) : undefined,parentContext: traceContext,},