OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: buffer performance
Carl-Gerhard Lindesvärd56336f2464a8
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex 5c948896a..a50ecf26a 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -115,6 +115,53 @@ export class BaseBuffer {return chunks;}+ /**+ * Max number of ch.insert sub-chunks that a single flush may run in+ * parallel. Caps connection-pool usage and worker memory peak. The+ * ClickHouse client pool defaults to 30 connections; with 3 workers and+ * up to ~3 buffers flushing concurrently cluster-wide, a per-flush cap+ * of 5 keeps total in-flight inserts well under the pool.+ */+ protected chInsertConcurrency = process.env.BUFFER_CH_INSERT_CONCURRENCY+ ? Math.max(1, Number.parseInt(process.env.BUFFER_CH_INSERT_CONCURRENCY, 10))+ : 5;++ /**+ * Run `fn` over `items` with a bounded concurrency. Preserves order of+ * `results`. If any worker throws, the helper propagates the first error+ * (consistent with `Promise.all`); other in-flight promises run to+ * completion but their resolved values/errors are discarded.+ */+ protected async parallelLimit<T, R>(+ items: T[],+ fn: (item: T, index: number) => Promise<R>,+ concurrency: number = this.chInsertConcurrency,+ ): Promise<R[]> {+ if (items.length === 0) return [];+ if (concurrency <= 1 || items.length === 1) {+ const out: R[] = [];+ for (let i = 0; i < items.length; i++) {+ out.push(await fn(items[i] as T, i));+ }+ return out;+ }+ const results = new Array<R>(items.length);+ let nextIndex = 0;+ const worker = async (): Promise<void> => {+ while (true) {+ const idx = nextIndex++;+ if (idx >= items.length) return;+ results[idx] = await fn(items[idx] as T, idx);+ }+ };+ const workers = Array.from(+ { length: Math.min(concurrency, items.length) },+ () => worker(),+ );+ await Promise.all(workers);+ return results;+ }+/*** Subclasses call this from within onFlush to record what they actually* did. The base class includes these in the FlushObservation it emits.diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex d486a6d4a..392d29ee1 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -169,13 +169,15 @@ export class EventBuffer extends BaseBuffer {);const chStart = performance.now();- for (const chunk of this.chunks(eventsToClickhouse, this.chunkSize)) {- await ch.insert({- table: 'events',- values: chunk,- format: 'JSONEachRow',- });- }+ await this.parallelLimit(+ this.chunks(eventsToClickhouse, this.chunkSize),+ (chunk) =>+ ch.insert({+ table: 'events',+ values: chunk,+ format: 'JSONEachRow',+ }),+ );const chInsertMs = performance.now() - chStart;const countByProject = new Map<string, number>();diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/db/src/buffers/group-buffer.tsindex 6fcfb6d8c..aa21ea7e3 100644--- a/packages/db/src/buffers/group-buffer.ts+++ b/packages/db/src/buffers/group-buffer.ts@@ -173,13 +173,13 @@ export class GroupBuffer extends BaseBuffer {const parsed = items.map((i) => getSafeJson<IGroupBufferEntry>(i));const chStart = performance.now();- for (const chunk of this.chunks(parsed, this.chunkSize)) {- await ch.insert({+ await this.parallelLimit(this.chunks(parsed, this.chunkSize), (chunk) =>+ ch.insert({table: TABLE_NAMES.groups,values: chunk,format: 'JSONEachRow',- });- }+ }),+ );const chInsertMs = performance.now() - chStart;const trimStart = performance.now();diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 171aad9e8..f3fc55f3a 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -336,13 +336,13 @@ export class ProfileBuffer extends BaseBuffer {}const chStart = performance.now();- for (const chunk of this.chunks(toInsert, this.chunkSize)) {- await ch.insert({+ await this.parallelLimit(this.chunks(toInsert, this.chunkSize), (chunk) =>+ ch.insert({table: TABLE_NAMES.profiles,values: chunk,format: 'JSONEachRow',- });- }+ }),+ );const chInsertMs = performance.now() - chStart;const trimStart = performance.now();diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/db/src/buffers/replay-buffer.tsindex 03112cd5c..af82b136c 100644--- a/packages/db/src/buffers/replay-buffer.ts+++ b/packages/db/src/buffers/replay-buffer.ts@@ -78,13 +78,13 @@ export class ReplayBuffer extends BaseBuffer {.filter((item): item is IClickhouseSessionReplayChunk => item != null);const chStart = performance.now();- for (const chunk of this.chunks(chunks, this.chunkSize)) {- await ch.insert({+ await this.parallelLimit(this.chunks(chunks, this.chunkSize), (chunk) =>+ ch.insert({table: TABLE_NAMES.session_replay_chunks,values: chunk,format: 'JSONEachRow',- });- }+ }),+ );const chInsertMs = performance.now() - chStart;const trimStart = performance.now();diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.tsindex 2e8c6a424..24ac5e94e 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/db/src/buffers/session-buffer.ts@@ -14,6 +14,10 @@ export class SessionBuffer extends BaseBuffer {? Number.parseInt(process.env.SESSION_BUFFER_CHUNK_SIZE, 10): 1000;+ private readonly squashEnabled =+ process.env.SESSION_BUFFER_SQUASH !== 'false' &&+ process.env.SESSION_BUFFER_SQUASH !== '0';+private readonly redisKey = 'session-buffer';private redis: Redis;constructor() {@@ -246,6 +250,80 @@ export class SessionBuffer extends BaseBuffer {});}+ /**+ * Squash multiple (sign=-1, sign=+1) pairs for the same session id into+ * just the boundary rows: the oldest -1 (cancels the previous CH state)+ * and the newest +1 (the final state). Single-row sessions (no update+ * pair yet) pass through unchanged.+ *+ * Correctness depends on the invariant maintained by `getSession`: every+ * update for an existing session emits exactly one -1 at version V and+ * one +1 at V+1, where V is the existing version. The oldest -1 in the+ * batch cancels CH's previous +1, and the newest +1 becomes the new+ * canonical row. Intermediate (-1, +1) pairs cancel each other in CH+ * either way — keeping them is wasted work.+ */+ private squashSessionsByVersion(+ sessions: IClickhouseSession[],+ ): IClickhouseSession[] {+ if (sessions.length <= 1) return sessions;++ const grouped = new Map<string, IClickhouseSession[]>();+ for (const s of sessions) {+ const key = s.id;+ const arr = grouped.get(key);+ if (arr) arr.push(s);+ else grouped.set(key, [s]);+ }++ let squashedCount = 0;+ const out: IClickhouseSession[] = [];++ for (const entries of grouped.values()) {+ if (entries.length === 1) {+ out.push(entries[0]!);+ continue;+ }+ let oldestNeg: IClickhouseSession | null = null;+ let newestPos: IClickhouseSession | null = null;+ for (const e of entries) {+ if (e.sign === -1) {+ if (!oldestNeg || e.version < oldestNeg.version) {+ oldestNeg = e;+ }+ } else if (e.sign === 1) {+ if (!newestPos || e.version > newestPos.version) {+ newestPos = e;+ }+ }+ }+ const emitted: IClickhouseSession[] = [];+ if (oldestNeg) emitted.push(oldestNeg);+ if (newestPos) emitted.push(newestPos);+ squashedCount += entries.length - emitted.length;+ out.push(...emitted);+ }++ if (squashedCount > 0) {+ this.logger.debug(+ {+ inputRows: sessions.length,+ outputRows: out.length,+ dropped: squashedCount,+ uniqueSessions: grouped.size,+ },+ 'Session batch squashed',+ );+ }++ console.log(+ squashedCount > 0 ? 'Session batch squashed' : 'Session batch not squashed',+ { inputRows: sessions.length, outputRows: out.length, dropped: squashedCount },+ );++ return out;+ }+async processBuffer() {const lrangeStart = performance.now();const events = await this.redis.lrange(@@ -260,23 +338,40 @@ export class SessionBuffer extends BaseBuffer {return;}- const sessions = events- .map((e) => getSafeJson<IClickhouseSession>(e))- .map((session) => {- return {- ...session,- duration: Math.max(0, session?.duration || 0),- };+ const parsed: IClickhouseSession[] = [];+ for (const raw of events) {+ const session = getSafeJson<IClickhouseSession>(raw);+ if (!session) continue;+ parsed.push({+ ...session,+ duration: Math.max(0, session.duration || 0),});+ }++ // A single high-activity session can produce many (sign=-1, sign=+1)+ // pairs within one flush window — each event for that session adds a+ // pair. The VersionedCollapsingMergeTree on `sessions` will collapse+ // them at merge time, so the final state is correct either way. But+ // inserting all the intermediate rows costs network bytes, gzip CPU,+ // CH ingest work, and CH merge work.+ //+ // We can squash within the batch and only insert the boundary rows:+ // the oldest sign=-1 (which cancels the previous CH state) and the+ // newest sign=+1 (the final state). Same end result, fewer rows.+ //+ // Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.+ const sessions = this.squashEnabled+ ? this.squashSessionsByVersion(parsed)+ : parsed;const chStart = performance.now();- for (const chunk of this.chunks(sessions, this.chunkSize)) {- await ch.insert({+ await this.parallelLimit(this.chunks(sessions, this.chunkSize), (chunk) =>+ ch.insert({table: TABLE_NAMES.sessions,values: chunk,format: 'JSONEachRow',- });- }+ }),+ );const chInsertMs = performance.now() - chStart;const trimStart = performance.now();diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 463bc31bf..0f78035f3 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -112,8 +112,28 @@ function getClickhouseSettings(): ClickHouseSettings {};}+// Request gzip is on by default — pays CPU on the worker to send less data+// over the wire. For payload-heavy buffers (replay) the gzip step happens on+// the single Node main thread and can block the event loop, so it's worth+// being able to flip it off to test. Set CLICKHOUSE_REQUEST_COMPRESSION=false+// to disable.+const requestCompressionEnabled =+ process.env.CLICKHOUSE_REQUEST_COMPRESSION !== 'false' &&+ process.env.CLICKHOUSE_REQUEST_COMPRESSION !== '0';++// Pool size per worker process. With parallel chunk inserts (cap 5 per+// flush) and a few buffers potentially flushing concurrently across cluster+// nodes, the previous 30 left headroom but might be tight at peak. Set+// CLICKHOUSE_MAX_OPEN_CONNECTIONS to override.+const maxOpenConnections = process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS+ ? Math.max(+ 1,+ Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10),+ )+ : 50;+export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {- max_open_connections: 30,+ max_open_connections: maxOpenConnections,request_timeout: 300_000,keep_alive: {enabled: true,@@ -122,7 +142,7 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {idle_socket_ttl: 7000,},compression: {- request: true,+ request: requestCompressionEnabled,},clickhouse_settings: getClickhouseSettings(),log: {diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.tsindex 0102999e3..4c71c3f12 100644--- a/packages/db/src/services/session.service.ts+++ b/packages/db/src/services/session.service.ts@@ -50,7 +50,10 @@ export interface IClickhouseSession {utm_content: string;utm_term: string;revenue: number;- sign: 1 | 0;+ // CollapsingMergeTree marker: +1 = current state, -1 = cancel a prior +1.+ // The session-buffer emits both +1 (new) and -1 (old) rows per update so+ // CH can collapse intermediate states.+ sign: 1 | -1;version: number;// Dynamically addedhas_replay?: boolean;