Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(ingestion): cut per-step instrumentation cost on the event hot path (#123908)

Paweł Ledwońffd0fb0a7a8e
diff --git a/nodejs/src/common/tracing/tracing-utils.test.ts b/nodejs/src/common/tracing/tracing-utils.test.ts
index 665490b6a891..35a3bb676c2b 100644
--- a/nodejs/src/common/tracing/tracing-utils.test.ts
+++ b/nodejs/src/common/tracing/tracing-utils.test.ts
@@ -1,4 +1,4 @@
-import { context, propagation, trace } from '@opentelemetry/api'
+import { ROOT_CONTEXT, TraceFlags, context, propagation, trace } from '@opentelemetry/api'
import { InMemorySpanExporter, NodeTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-node'
import { register } from 'prom-client'
@@ -51,6 +51,23 @@ describe('instrumentFn', () => {
expect(byName.step.parentSpanContext?.spanId).toBe(byName.batch.spanContext().spanId)
})
+ it('runs work under an unsampled parentContext inside that context, so nested spans stay unsampled', async () => {
+ const unsampledParent = trace.setSpanContext(ROOT_CONTEXT, {
+ traceId: '0af7651916cd43dd8448eb211c80319c',
+ spanId: 'b7ad6b7169203331',
+ traceFlags: TraceFlags.NONE,
+ })
+
+ await instrumentFn({ key: 'step', parentContext: unsampledParent }, () =>
+ trace.getTracer('test').startActiveSpan('nested', (nested) => {
+ nested.end()
+ return Promise.resolve()
+ })
+ )
+
+ expect(exporter.getFinishedSpans()).toEqual([])
+ })
+
it('captures a failure while its span is active, so the error carries its trace id', async () => {
let activeSpanIdAtCapture: string | undefined
jest.spyOn(posthogUtils, 'captureException').mockImplementationOnce(() => {
diff --git a/nodejs/src/common/tracing/tracing-utils.ts b/nodejs/src/common/tracing/tracing-utils.ts
index ac8a69ec0587..4fa8701723d9 100644
--- a/nodejs/src/common/tracing/tracing-utils.ts
+++ b/nodejs/src/common/tracing/tracing-utils.ts
@@ -1,4 +1,16 @@
-import { Attributes, Context, HrTime, Span, SpanKind, SpanStatusCode, Tracer, context, trace } from '@opentelemetry/api'
+import {
+ Attributes,
+ Context,
+ HrTime,
+ Span,
+ SpanKind,
+ SpanStatusCode,
+ TraceFlags,
+ Tracer,
+ context,
+ isSpanContextValid,
+ trace,
+} from '@opentelemetry/api'
import { Counter, Histogram, Summary, exponentialBuckets } from 'prom-client'
import { defaultConfig } from '~/common/config/config'
@@ -44,6 +56,13 @@ function getHighResTimestamp(): HrTime {
const nanos = Math.round((epochMillis % 1000) * 1_000_000)
return [seconds, nanos]
}
+
+// The sampler is parent-based, so a child of an unsampled span is never exported.
+function hasUnsampledParent(ctx: Context): boolean {
+ const parent = trace.getSpanContext(ctx)
+ return parent !== undefined && isSpanContextValid(parent) && (parent.traceFlags & TraceFlags.SAMPLED) === 0
+}
+
/**
* Wraps a function in an OpenTelemetry tracing span.
*/
@@ -54,12 +73,16 @@ export function withTracingSpan<T>(
fn: () => Promise<T>,
parentContext?: Context
): Promise<T> {
+ const ctx = parentContext ?? context.active()
+ if (hasUnsampledParent(ctx)) {
+ return parentContext ? context.with(parentContext, fn) : fn()
+ }
const _tracer = typeof tracer === 'string' ? trace.getTracer(tracer) : tracer
const startHrTime = getHighResTimestamp()
return _tracer.startActiveSpan(
name,
{ kind: SpanKind.CLIENT, attributes: attrs, startTime: startHrTime },
- parentContext ?? context.active(),
+ ctx,
async (span) => {
try {
const out = await fn()
@@ -137,6 +160,8 @@ interface FunctionInstrumentationOptions {
parentContext?: Context
/** False keeps the timeout guard and both duration metrics but emits no span. */
span?: boolean
+ /** False skips the `instrumented_fn_duration_ms` summary, for callers that record their own duration. */
+ summary?: boolean
}
/**
@@ -182,6 +207,7 @@ export async function instrumentFn<T>(
const attributes = (typeof options === 'string' ? undefined : options.attributes) ?? {}
const parentContext = typeof options === 'string' ? undefined : options.parentContext
const span = (typeof options === 'string' ? undefined : options.span) ?? true
+ const summary = (typeof options === 'string' ? undefined : options.summary) ?? true
const t = timeoutGuard(timeoutMessage, getLoggingContext, timeout, sendException, () => {
instrumentedFunctionTimeout.labels({ function: key }).inc()
@@ -201,17 +227,19 @@ export async function instrumentFn<T>(
try {
// Skip expensive span creation when tracing is disabled
- const result = defaultConfig.DISABLE_OPENTELEMETRY_TRACING
- ? await funcReportingErrorsInSpan()
- : span
- ? await withSpan(
- 'instrumented_function',
- key,
- tag ? { ...attributes, tag } : attributes,
- funcReportingErrorsInSpan,
- parentContext
- )
- : await withSummary(key, tag, funcReportingErrorsInSpan)
+ const tracingEnabled = !defaultConfig.DISABLE_OPENTELEMETRY_TRACING
+ const run =
+ tracingEnabled && span
+ ? () =>
+ withTracingSpan(
+ 'instrumented_function',
+ key,
+ tag ? { ...attributes, tag } : attributes,
+ funcReportingErrorsInSpan,
+ parentContext
+ )
+ : funcReportingErrorsInSpan
+ const result = tracingEnabled && summary ? await withSummary(key, tag, run) : await run()
end?.({ success: 'true' })
if (logExecutionTime) {
logTime(startTime, key)
diff --git a/nodejs/src/common/utils/lazy-loader.ts b/nodejs/src/common/utils/lazy-loader.ts
index 880e178132fa..0e5d72431b01 100644
--- a/nodejs/src/common/utils/lazy-loader.ts
+++ b/nodejs/src/common/utils/lazy-loader.ts
@@ -233,56 +233,57 @@ export class LazyLoader<T> {
* If the value is older than the refreshAge, it is loaded from the database.
*/
private async loadViaCache(keys: string[], options?: LoadOptions): Promise<Record<string, T | null>> {
- return await instrumentFn({ key: `lazyLoader.loadViaCache`, tag: this.options.name, span: false }, async () => {
- // No prototype, for the same reason as the cache: keys are caller-supplied, and this
- // object is handed back to callers who may iterate or spread it.
- const results: Record<string, T | null> = Object.create(null)
- const keysToLoad = new Set<string>()
-
- for (const key of keys) {
- const cached = this.cache[key]
-
- if (cached !== undefined) {
- results[key] = cached.value
- cached.lastUsed = Date.now()
-
- const cacheUntil = cached.cacheUntil
- const backgroundRefreshAfter = cached.backgroundRefreshAfter
-
- if (Date.now() > cacheUntil) {
- keysToLoad.add(key)
- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
- continue
- }
+ // No prototype, for the same reason as the cache: keys are caller-supplied, and this
+ // object is handed back to callers who may iterate or spread it.
+ const results: Record<string, T | null> = Object.create(null)
+ const keysToLoad = new Set<string>()
- if (backgroundRefreshAfter && Date.now() > backgroundRefreshAfter) {
- void this.load([key]).catch((err) => {
- logger.warn(`[LazyLoader:${this.options.name}] Background refresh failed`, {
- key,
- error: String(err),
- })
- })
- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit_background' }).inc()
- continue
- }
- } else {
+ for (const key of keys) {
+ const cached = this.cache[key]
+
+ if (cached !== undefined) {
+ results[key] = cached.value
+ cached.lastUsed = Date.now()
+
+ const cacheUntil = cached.cacheUntil
+ const backgroundRefreshAfter = cached.backgroundRefreshAfter
+
+ if (Date.now() > cacheUntil) {
keysToLoad.add(key)
lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
continue
}
- lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
+ if (backgroundRefreshAfter && Date.now() > backgroundRefreshAfter) {
+ void this.load([key]).catch((err) => {
+ logger.warn(`[LazyLoader:${this.options.name}] Background refresh failed`, {
+ key,
+ error: String(err),
+ })
+ })
+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit_background' }).inc()
+ continue
+ }
+ } else {
+ keysToLoad.add(key)
+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
+ continue
}
- if (keysToLoad.size === 0) {
- lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
- return results
- }
+ lazyLoaderCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
+ }
+
+ if (keysToLoad.size === 0) {
+ lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'hit' }).inc()
+ return results
+ }
- lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
+ lazyLoaderFullCacheHits.labels({ name: this.options.name, hit: 'miss' }).inc()
- // Only a miss gets a span: with one span per lookup, hits outnumber every other span in a trace.
- return await withTracingSpan(
+ // Only a miss is instrumented: hits would outnumber every other span in a trace, and on a hit the
+ // instrumentation costs more than the lookup.
+ return await instrumentFn({ key: `lazyLoader.loadViaCache`, tag: this.options.name, span: false }, () =>
+ withTracingSpan(
'instrumented_function',
'lazyLoader.loadViaCache',
{ tag: this.options.name },
@@ -301,7 +302,7 @@ export class LazyLoader<T> {
return results
}
)
- })
+ )
}
private spanAttributes(keys: string[], keysToLoad: Set<string>, outcome: LazyLoaderSpanOutcome): Attributes {
diff --git a/nodejs/src/ingestion/framework/base-chunk-pipeline.ts b/nodejs/src/ingestion/framework/base-chunk-pipeline.ts
index 94960094dfed..93b184bc70f4 100644
--- a/nodejs/src/ingestion/framework/base-chunk-pipeline.ts
+++ b/nodejs/src/ingestion/framework/base-chunk-pipeline.ts
@@ -82,7 +82,7 @@ export async function applyChunkStepToResults<TIn, TOut, C, RPrev extends string
const end = pipelineStepDurationHistogram.startTimer({ step_name: stepName, step_type: 'chunk' })
try {
stepResults = await instrumentFn(
- { key: stepName, sendException: false, measureTime: false, attributes, parentContext },
+ { key: stepName, sendException: false, measureTime: false, summary: false, attributes, parentContext },
() => step(successfulValues)
)
end({ result: 'chunk' })
diff --git a/nodejs/src/ingestion/framework/step-pipeline.ts b/nodejs/src/ingestion/framework/step-pipeline.ts
index 7a5c9f529fa6..fe204d6f5a64 100644
--- a/nodejs/src/ingestion/framework/step-pipeline.ts
+++ b/nodejs/src/ingestion/framework/step-pipeline.ts
@@ -57,6 +57,7 @@ export class StepPipeline<TInput, TIntermediate, TOutput, C, RPrev extends strin
key: this.stepName,
sendException: false,
measureTime: false,
+ summary: false,
attributes: isTracingActive(traceContext) ? eventSpanAttributes(previousResult.value) : undefined,
parentContext: traceContext,
},