Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: optimize profile buffer and add metrics

Carl-Gerhard Lindesvärd515377f00c96
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.ts
index 6d8c3743a..00839945d 100644
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -174,6 +174,18 @@ const chInsertDurationMs = new client.Histogram({
});
register.registerMetric(chInsertDurationMs);
+// CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing
+// profiles for merge). Separated from ch_insert because for some buffers
+// the SELECT dominates total flush time. Only populated by buffers that
+// actually read CH inside the flush path.
+const chFetchDurationMs = new client.Histogram({
+ name: 'buffer_ch_fetch_duration_ms',
+ help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',
+ labelNames: ['buffer'],
+ buckets: [10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000],
+});
+register.registerMetric(chFetchDurationMs);
+
const flushLlenAtStart = new client.Histogram({
name: 'buffer_flush_llen_at_start',
help: 'LLEN observed at the start of a flush attempt',
@@ -194,11 +206,18 @@ register.registerMetric(addDurationMs);
const addTotal = new client.Counter({
name: 'buffer_add_total',
- help: 'Total add() calls per buffer (ingest rate). Pair with buffer_flush_rows_total for fill-vs-drain.',
+ help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',
labelNames: ['buffer'],
});
register.registerMetric(addTotal);
+const addSkippedTotal = new client.Counter({
+ name: 'buffer_add_skipped_total',
+ help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',
+ labelNames: ['buffer', 'reason'],
+});
+register.registerMetric(addSkippedTotal);
+
for (const buf of allBuffers) {
buf.flushObserver = (obs) => {
flushTotal.inc({
@@ -231,6 +250,12 @@ for (const buf of allBuffers) {
obs.phases.trimMs,
);
}
+ if (obs.phases?.chFetchMs != null) {
+ chFetchDurationMs.observe(
+ { buffer: obs.buffer },
+ obs.phases.chFetchMs,
+ );
+ }
if (obs.phases?.chInsertMs != null) {
chInsertDurationMs.observe(
{ buffer: obs.buffer },
@@ -240,6 +265,14 @@ for (const buf of allBuffers) {
};
buf.addObserver = (obs) => {
+ if (obs.skipped) {
+ addSkippedTotal.inc({
+ buffer: obs.buffer,
+ reason: obs.skipReason ?? 'unknown',
+ });
+ // Don't pollute add-latency histogram with no-op fast paths
+ return;
+ }
addTotal.inc({ buffer: obs.buffer });
addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);
};
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts
index daf183eb7..5c948896a 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/db/src/buffers/base-buffer.ts
@@ -5,6 +5,10 @@ import { getRedisCache } from '@openpanel/redis';
export type FlushPhaseTimings = {
lrangeMs?: number;
+ /** Time spent in CH SELECT(s) during flush (e.g. profile-buffer's
+ * fetch-existing-profile-for-merge step). Only buffers that read CH
+ * during flush will populate this. */
+ chFetchMs?: number;
chInsertMs?: number;
trimMs?: number;
/** Total time spent inside onFlush (subclass-specific processing) */
@@ -39,6 +43,12 @@ export type FlushObserver = (obs: FlushObservation) => void;
export type AddObservation = {
buffer: string;
durationMs: number;
+ /** True if the add was a no-op (e.g. cached-and-skipped). When set,
+ * prefer not to charge this against add-latency histograms — the call
+ * did almost no work. */
+ skipped?: boolean;
+ /** Free-form reason label for skipped adds (e.g. 'cached'). */
+ skipReason?: string;
};
export type AddObserver = (obs: AddObservation) => void;
@@ -61,6 +71,13 @@ export class BaseBuffer {
phases?: FlushPhaseTimings;
} = {};
+ /** Subclass-set markers for the in-flight add() call. Reset on every
+ * call so a single short-circuit doesn't pollute the next observation. */
+ private inflightAddStats: {
+ skipped?: boolean;
+ skipReason?: string;
+ } = {};
+
/** Optional hook used by the worker to bridge flushes into Prometheus. */
public flushObserver: FlushObserver | null = null;
@@ -131,9 +148,12 @@ export class BaseBuffer {
/**
* Time the body of an `add()` call. Subclasses wrap their work with this
* so all add latency is reported uniformly via the `addObserver` hook.
+ * Subclasses may call `reportAddSkipped(reason)` from inside the fn to
+ * mark the call as a no-op (e.g. cached-and-skipped).
*/
protected async timeAdd<T>(fn: () => Promise<T>): Promise<T> {
const start = performance.now();
+ this.inflightAddStats = {};
try {
return await fn();
} finally {
@@ -141,6 +161,8 @@ export class BaseBuffer {
this.addObserver?.({
buffer: this.name,
durationMs: performance.now() - start,
+ skipped: this.inflightAddStats.skipped,
+ skipReason: this.inflightAddStats.skipReason,
});
} catch {
// observer errors must not break add
@@ -148,6 +170,12 @@ export class BaseBuffer {
}
}
+ /** Mark the in-flight add() as a no-op for observability. */
+ protected reportAddSkipped(reason: string) {
+ this.inflightAddStats.skipped = true;
+ this.inflightAddStats.skipReason = reason;
+ }
+
private async releaseLock(lockId: string): Promise<void> {
const script = `
if redis.call("get", KEYS[1]) == ARGV[1] then
@@ -181,6 +209,10 @@ export class BaseBuffer {
obs.phases?.lrangeMs != null
? Math.round(obs.phases.lrangeMs)
: undefined,
+ chFetchMs:
+ obs.phases?.chFetchMs != null
+ ? Math.round(obs.phases.chFetchMs)
+ : undefined,
chInsertMs:
obs.phases?.chInsertMs != null
? Math.round(obs.phases.chInsertMs)
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index 63d2e2421..171aad9e8 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -13,6 +13,35 @@ import { BaseBuffer } from './base-buffer';
const PROFILE_COLUMNS =
'id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, last_seen_at, groups';
+// Equivalent of `SELECT ... FROM profiles FINAL` but expressed via GROUP BY +
+// argMax. ReplicatedReplacingMergeTree uses `last_seen_at` as the version
+// column, so argMax(col, last_seen_at) yields the same value FINAL would
+// produce. Avoiding FINAL lets CH use the primary key (project_id, id) to
+// scan instead of merging parts at query time — order(s) of magnitude
+// faster on the profile-buffer's fetch path.
+//
+// `p.last_seen_at` is qualified with the FROM-clause table alias because
+// the SELECT exposes a `last_seen_at` aggregate alias, and CH resolves
+// bare column refs against the alias list first — causing ILLEGAL_AGGREGATION
+// inside the argMax/max calls. Qualifying with `p.` bypasses the alias
+// lookup and binds to the raw column.
+const PROFILE_LATEST_AGGREGATE_COLUMNS = [
+ 'id',
+ 'project_id',
+ 'argMax(first_name, p.last_seen_at) AS first_name',
+ 'argMax(last_name, p.last_seen_at) AS last_name',
+ 'argMax(email, p.last_seen_at) AS email',
+ 'argMax(avatar, p.last_seen_at) AS avatar',
+ 'argMax(properties, p.last_seen_at) AS properties',
+ 'argMax(is_external, p.last_seen_at) AS is_external',
+ 'argMax(groups, p.last_seen_at) AS groups',
+ // created_at doesn't change between rows for the same profile, but
+ // min() is safe and matches "first seen" semantics if there ever is a
+ // discrepancy.
+ 'min(created_at) AS created_at',
+ 'max(p.last_seen_at) AS last_seen_at',
+].join(', ');
+
export class ProfileBuffer extends BaseBuffer {
private readonly batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE
? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)
@@ -88,6 +117,7 @@ export class ProfileBuffer extends BaseBuffer {
});
const exists = await this.redis.exists(cacheKey);
if (exists === 1) {
+ this.reportAddSkipped('cached');
return;
}
}
@@ -179,11 +209,17 @@ export class ProfileBuffer extends BaseBuffer {
)
.join(', ');
try {
+ // Table alias `p` is required: without it, WHERE's `last_seen_at`
+ // resolves to the SELECT-list aggregate alias `max(last_seen_at) AS
+ // last_seen_at`, which is an aggregate function and illegal in WHERE
+ // (CH ILLEGAL_AGGREGATION). Qualifying with `p.` bypasses the alias
+ // lookup and binds to the raw column.
const rows = await chQuery<IClickhouseProfile>(
- `SELECT ${PROFILE_COLUMNS}
- FROM ${TABLE_NAMES.profiles} FINAL
- WHERE (id, project_id) IN (${tuples})
- ${withDateFilter ? `AND ${TABLE_NAMES.profiles}.last_seen_at > now() - INTERVAL 2 DAY` : ''}`
+ `SELECT ${PROFILE_LATEST_AGGREGATE_COLUMNS}
+ FROM ${TABLE_NAMES.profiles} AS p
+ WHERE (p.id, p.project_id) IN (${tuples})
+ ${withDateFilter ? `AND p.last_seen_at > now() - INTERVAL 2 DAY` : ''}
+ GROUP BY p.id, p.project_id`
);
for (const row of rows) {
result.set(`${row.project_id}:${row.id}`, row);
@@ -262,10 +298,16 @@ export class ProfileBuffer extends BaseBuffer {
}
}
- // Fetch cache misses from ClickHouse in bounded chunks
+ // Fetch cache misses from ClickHouse in bounded chunks. Timed separately
+ // from the ch.insert phase because it can dominate flush time for
+ // profile-buffer specifically (FINAL replaced with argMax + GROUP BY in
+ // batchFetchFromClickhouse, but worth measuring directly).
+ let chFetchMs: number | undefined;
if (cacheMisses.length > 0) {
+ const chFetchStart = performance.now();
const clickhouseResults =
await this.batchFetchFromClickhouse(cacheMisses);
+ chFetchMs = performance.now() - chFetchStart;
for (const [key, profile] of clickhouseResults) {
existingByKey.set(key, profile);
}
@@ -310,7 +352,7 @@ export class ProfileBuffer extends BaseBuffer {
this.reportFlushStats({
rowsProcessed: rawProfiles.length,
- phases: { lrangeMs, chInsertMs, trimMs },
+ phases: { lrangeMs, chFetchMs, chInsertMs, trimMs },
});
}
}
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index 17775b735..4a393c4d7 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -424,16 +424,23 @@ export async function createEvent(payload: IServiceCreateEventPayload) {
};
// Only upsert the profile on session boundaries.
- // - session_start covers fresh activity (drives `last_seen_at` via
- // `created_at`, our ReplacingMergeTree version column).
- // - session_end is synthesized server-side by the worker, so we get a
- // reliable end-of-session bump even when the SDK doesn't fire it.
- // Per-event upserts for identified users are skipped — `op.identify()` /
- // `op.setProfile()` go through the controller path directly and stay fresh.
- // `isFromEvent=false` so the buffer's cache-shortcut doesn't suppress the
- // write — we want every session boundary to bump `last_seen_at`.
+ // - session_start covers fresh activity.
+ // - session_end is synthesized server-side by the worker.
+ // Identified users' explicit profile writes (op.identify(), op.setProfile())
+ // go through the controller path and are not affected by this branch.
+ //
+ // `isFromEvent=true` activates profile-buffer's cache shortcut: if the
+ // profile is in the 1h Redis cache (i.e. recently flushed), the add is
+ // skipped. Trade-off: profile.last_seen_at granularity is capped at the
+ // cache TTL (~1h) rather than per-session. We accept this because
+ // (a) profile-buffer was the leading indicator in the 2026-05-20 buildup
+ // and was processing ~2 writes per session per anonymous user;
+ // (b) the bulk of those writes carried no new information (anonymous
+ // profile data is event-derived and stable across a session);
+ // (c) recency queries should derive from event timestamps, not from
+ // profile.last_seen_at.
if (payload.name === 'session_start' || payload.name === 'session_end') {
- promises.push(upsertProfile(profile));
+ promises.push(upsertProfile(profile, true));
}
}