OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: optimize profile buffer and add metrics
Carl-Gerhard Lindesvärd515377f00c96
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex 6d8c3743a..00839945d 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -174,6 +174,18 @@ const chInsertDurationMs = new client.Histogram({});register.registerMetric(chInsertDurationMs);+// CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing+// profiles for merge). Separated from ch_insert because for some buffers+// the SELECT dominates total flush time. Only populated by buffers that+// actually read CH inside the flush path.+const chFetchDurationMs = new client.Histogram({+ name: 'buffer_ch_fetch_duration_ms',+ help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',+ labelNames: ['buffer'],+ buckets: [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000],+});+register.registerMetric(chFetchDurationMs);+const flushLlenAtStart = new client.Histogram({name: 'buffer_flush_llen_at_start',help: 'LLEN observed at the start of a flush attempt',@@ -194,11 +206,18 @@ register.registerMetric(addDurationMs);const addTotal = new client.Counter({name: 'buffer_add_total',- help: 'Total add() calls per buffer (ingest rate). Pair with buffer_flush_rows_total for fill-vs-drain.',+ help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',labelNames: ['buffer'],});register.registerMetric(addTotal);+const addSkippedTotal = new client.Counter({+ name: 'buffer_add_skipped_total',+ help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',+ labelNames: ['buffer', 'reason'],+});+register.registerMetric(addSkippedTotal);+for (const buf of allBuffers) {buf.flushObserver = (obs) => {flushTotal.inc({@@ -231,6 +250,12 @@ for (const buf of allBuffers) {obs.phases.trimMs,);}+ if (obs.phases?.chFetchMs != null) {+ chFetchDurationMs.observe(+ { buffer: obs.buffer },+ obs.phases.chFetchMs,+ );+ }if (obs.phases?.chInsertMs != null) {chInsertDurationMs.observe({ buffer: obs.buffer },@@ -240,6 +265,14 @@ for (const buf of allBuffers) {};buf.addObserver = (obs) => {+ if (obs.skipped) {+ addSkippedTotal.inc({+ buffer: obs.buffer,+ reason: obs.skipReason ?? 'unknown',+ });+ // Don't pollute add-latency histogram with no-op fast paths+ return;+ }addTotal.inc({ buffer: obs.buffer });addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);};diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex daf183eb7..5c948896a 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -5,6 +5,10 @@ import { getRedisCache } from '@openpanel/redis';export type FlushPhaseTimings = {lrangeMs?: number;+ /** Time spent in CH SELECT(s) during flush (e.g. profile-buffer's+ * fetch-existing-profile-for-merge step). Only buffers that read CH+ * during flush will populate this. */+ chFetchMs?: number;chInsertMs?: number;trimMs?: number;/** Total time spent inside onFlush (subclass-specific processing) */@@ -39,6 +43,12 @@ export type FlushObserver = (obs: FlushObservation) => void;export type AddObservation = {buffer: string;durationMs: number;+ /** True if the add was a no-op (e.g. cached-and-skipped). When set,+ * prefer not to charge this against add-latency histograms — the call+ * did almost no work. */+ skipped?: boolean;+ /** Free-form reason label for skipped adds (e.g. 'cached'). */+ skipReason?: string;};export type AddObserver = (obs: AddObservation) => void;@@ -61,6 +71,13 @@ export class BaseBuffer {phases?: FlushPhaseTimings;} = {};+ /** Subclass-set markers for the in-flight add() call. Reset on every+ * call so a single short-circuit doesn't pollute the next observation. */+ private inflightAddStats: {+ skipped?: boolean;+ skipReason?: string;+ } = {};+/** Optional hook used by the worker to bridge flushes into Prometheus. */public flushObserver: FlushObserver | null = null;@@ -131,9 +148,12 @@ export class BaseBuffer {/*** Time the body of an `add()` call. Subclasses wrap their work with this* so all add latency is reported uniformly via the `addObserver` hook.+ * Subclasses may call `reportAddSkipped(reason)` from inside the fn to+ * mark the call as a no-op (e.g. cached-and-skipped).*/protected async timeAdd<T>(fn: () => Promise<T>): Promise<T> {const start = performance.now();+ this.inflightAddStats = {};try {return await fn();} finally {@@ -141,6 +161,8 @@ export class BaseBuffer {this.addObserver?.({buffer: this.name,durationMs: performance.now() - start,+ skipped: this.inflightAddStats.skipped,+ skipReason: this.inflightAddStats.skipReason,});} catch {// observer errors must not break add@@ -148,6 +170,12 @@ export class BaseBuffer {}}+ /** Mark the in-flight add() as a no-op for observability. */+ protected reportAddSkipped(reason: string) {+ this.inflightAddStats.skipped = true;+ this.inflightAddStats.skipReason = reason;+ }+private async releaseLock(lockId: string): Promise<void> {const script = `if redis.call("get", KEYS[1]) == ARGV[1] then@@ -181,6 +209,10 @@ export class BaseBuffer {obs.phases?.lrangeMs != null? Math.round(obs.phases.lrangeMs): undefined,+ chFetchMs:+ obs.phases?.chFetchMs != null+ ? Math.round(obs.phases.chFetchMs)+ : undefined,chInsertMs:obs.phases?.chInsertMs != null? Math.round(obs.phases.chInsertMs)diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 63d2e2421..171aad9e8 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -13,6 +13,35 @@ import { BaseBuffer } from './base-buffer';const PROFILE_COLUMNS ='id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, last_seen_at, groups';+// Equivalent of `SELECT ... FROM profiles FINAL` but expressed via GROUP BY ++// argMax. ReplicatedReplacingMergeTree uses `last_seen_at` as the version+// column, so argMax(col, last_seen_at) yields the same value FINAL would+// produce. Avoiding FINAL lets CH use the primary key (project_id, id) to+// scan instead of merging parts at query time — order(s) of magnitude+// faster on the profile-buffer's fetch path.+//+// `p.last_seen_at` is qualified with the FROM-clause table alias because+// the SELECT exposes a `last_seen_at` aggregate alias, and CH resolves+// bare column refs against the alias list first — causing ILLEGAL_AGGREGATION+// inside the argMax/max calls. Qualifying with `p.` bypasses the alias+// lookup and binds to the raw column.+const PROFILE_LATEST_AGGREGATE_COLUMNS = [+ 'id',+ 'project_id',+ 'argMax(first_name, p.last_seen_at) AS first_name',+ 'argMax(last_name, p.last_seen_at) AS last_name',+ 'argMax(email, p.last_seen_at) AS email',+ 'argMax(avatar, p.last_seen_at) AS avatar',+ 'argMax(properties, p.last_seen_at) AS properties',+ 'argMax(is_external, p.last_seen_at) AS is_external',+ 'argMax(groups, p.last_seen_at) AS groups',+ // created_at doesn't change between rows for the same profile, but+ // min() is safe and matches "first seen" semantics if there ever is a+ // discrepancy.+ 'min(created_at) AS created_at',+ 'max(p.last_seen_at) AS last_seen_at',+].join(', ');+export class ProfileBuffer extends BaseBuffer {private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)@@ -88,6 +117,7 @@ export class ProfileBuffer extends BaseBuffer {});const exists = await this.redis.exists(cacheKey);if (exists === 1) {+ this.reportAddSkipped('cached');return;}}@@ -179,11 +209,17 @@ export class ProfileBuffer extends BaseBuffer {).join(', ');try {+ // Table alias `p` is required: without it, WHERE's `last_seen_at`+ // resolves to the SELECT-list aggregate alias `max(last_seen_at) AS+ // last_seen_at`, which is an aggregate function and illegal in WHERE+ // (CH ILLEGAL_AGGREGATION). Qualifying with `p.` bypasses the alias+ // lookup and binds to the raw column.const rows = await chQuery<IClickhouseProfile>(- `SELECT ${PROFILE_COLUMNS}- FROM ${TABLE_NAMES.profiles} FINAL- WHERE (id, project_id) IN (${tuples})- ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.last_seen_at > now() - INTERVAL 2 DAY` : ''}`+ `SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}+ FROM ${TABLE_NAMES.profiles} AS p+ WHERE (p.id, p.project_id) IN (${tuples})+ ${withDateFilter ? `AND p.last_seen_at > now() - INTERVAL 2 DAY` : ''}+ GROUP BY p.id, p.project_id`);for (const row of rows) {result.set(`${row.project_id}:${row.id}`, row);@@ -262,10 +298,16 @@ export class ProfileBuffer extends BaseBuffer {}}- // Fetch cache misses from ClickHouse in bounded chunks+ // Fetch cache misses from ClickHouse in bounded chunks. Timed separately+ // from the ch.insert phase because it can dominate flush time for+ // profile-buffer specifically (FINAL replaced with argMax + GROUP BY in+ // batchFetchFromClickhouse, but worth measuring directly).+ let chFetchMs: number | undefined;if (cacheMisses.length > 0) {+ const chFetchStart = performance.now();const clickhouseResults =await this.batchFetchFromClickhouse(cacheMisses);+ chFetchMs = performance.now() - chFetchStart;for (const [key, profile] of clickhouseResults) {existingByKey.set(key, profile);}@@ -310,7 +352,7 @@ export class ProfileBuffer extends BaseBuffer {this.reportFlushStats({rowsProcessed: rawProfiles.length,- phases: { lrangeMs, chInsertMs, trimMs },+ phases: { lrangeMs, chFetchMs, chInsertMs, trimMs },});}}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 17775b735..4a393c4d7 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -424,16 +424,23 @@ export async function createEvent(payload: IServiceCreateEventPayload) {};// Only upsert the profile on session boundaries.- // - session_start covers fresh activity (drives `last_seen_at` via- // `created_at`, our ReplacingMergeTree version column).- // - session_end is synthesized server-side by the worker, so we get a- // reliable end-of-session bump even when the SDK doesn't fire it.- // Per-event upserts for identified users are skipped — `op.identify()` /- // `op.setProfile()` go through the controller path directly and stay fresh.- // `isFromEvent=false` so the buffer's cache-shortcut doesn't suppress the- // write — we want every session boundary to bump `last_seen_at`.+ // - session_start covers fresh activity.+ // - session_end is synthesized server-side by the worker.+ // Identified users' explicit profile writes (op.identify(), op.setProfile())+ // go through the controller path and are not affected by this branch.+ //+ // `isFromEvent=true` activates profile-buffer's cache shortcut: if the+ // profile is in the 1h Redis cache (i.e. recently flushed), the add is+ // skipped. Trade-off: profile.last_seen_at granularity is capped at the+ // cache TTL (~1h) rather than per-session. We accept this because+ // (a) profile-buffer was the leading indicator in the 2026-05-20 buildup+ // and was processing ~2 writes per session per anonymous user;+ // (b) the bulk of those writes carried no new information (anonymous+ // profile data is event-derived and stable across a session);+ // (c) recency queries should derive from event timestamps, not from+ // profile.last_seen_at.if (payload.name === 'session_start' || payload.name === 'session_end') {- promises.push(upsertProfile(profile));+ promises.push(upsertProfile(profile, true));}}