Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: buffer performance

Carl-Gerhard Lindesvärd56336f2464a8
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.ts
index 5c948896a..a50ecf26a 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/db/src/buffers/base-buffer.ts
@@ -115,6 +115,53 @@ export class BaseBuffer {
return chunks;
}
+ /**
+ * Max number of ch.insert sub-chunks that a single flush may run in
+ * parallel. Caps connection-pool usage and worker memory peak. The
+ * ClickHouse client pool defaults to 30 connections; with 3 workers and
+ * up to ~3 buffers flushing concurrently cluster-wide, a per-flush cap
+ * of 5 keeps total in-flight inserts well under the pool.
+ */
+ protected chInsertConcurrency = process.env.BUFFER_CH_INSERT_CONCURRENCY
+ ? Math.max(1, Number.parseInt(process.env.BUFFER_CH_INSERT_CONCURRENCY, 10))
+ : 5;
+
+ /**
+ * Run `fn` over `items` with a bounded concurrency. Preserves order of
+ * `results`. If any worker throws, the helper propagates the first error
+ * (consistent with `Promise.all`); other in-flight promises run to
+ * completion but their resolved values/errors are discarded.
+ */
+ protected async parallelLimit<T, R>(
+ items: T[],
+ fn: (item: T, index: number) => Promise<R>,
+ concurrency: number = this.chInsertConcurrency,
+ ): Promise<R[]> {
+ if (items.length === 0) return [];
+ if (concurrency <= 1 || items.length === 1) {
+ const out: R[] = [];
+ for (let i = 0; i < items.length; i++) {
+ out.push(await fn(items[i] as T, i));
+ }
+ return out;
+ }
+ const results = new Array<R>(items.length);
+ let nextIndex = 0;
+ const worker = async (): Promise<void> => {
+ while (true) {
+ const idx = nextIndex++;
+ if (idx >= items.length) return;
+ results[idx] = await fn(items[idx] as T, idx);
+ }
+ };
+ const workers = Array.from(
+ { length: Math.min(concurrency, items.length) },
+ () => worker(),
+ );
+ await Promise.all(workers);
+ return results;
+ }
+
/**
* Subclasses call this from within onFlush to record what they actually
* did. The base class includes these in the FlushObservation it emits.
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.ts
index d486a6d4a..392d29ee1 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/db/src/buffers/event-buffer.ts
@@ -169,13 +169,15 @@ export class EventBuffer extends BaseBuffer {
);
const chStart = performance.now();
- for (const chunk of this.chunks(eventsToClickhouse, this.chunkSize)) {
- await ch.insert({
- table: 'events',
- values: chunk,
- format: 'JSONEachRow',
- });
- }
+ await this.parallelLimit(
+ this.chunks(eventsToClickhouse, this.chunkSize),
+ (chunk) =>
+ ch.insert({
+ table: 'events',
+ values: chunk,
+ format: 'JSONEachRow',
+ }),
+ );
const chInsertMs = performance.now() - chStart;
const countByProject = new Map<string, number>();
diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/db/src/buffers/group-buffer.ts
index 6fcfb6d8c..aa21ea7e3 100644
--- a/packages/db/src/buffers/group-buffer.ts
+++ b/packages/db/src/buffers/group-buffer.ts
@@ -173,13 +173,13 @@ export class GroupBuffer extends BaseBuffer {
const parsed = items.map((i) => getSafeJson<IGroupBufferEntry>(i));
const chStart = performance.now();
- for (const chunk of this.chunks(parsed, this.chunkSize)) {
- await ch.insert({
+ await this.parallelLimit(this.chunks(parsed, this.chunkSize), (chunk) =>
+ ch.insert({
table: TABLE_NAMES.groups,
values: chunk,
format: 'JSONEachRow',
- });
- }
+ }),
+ );
const chInsertMs = performance.now() - chStart;
const trimStart = performance.now();
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.ts
index 171aad9e8..f3fc55f3a 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/db/src/buffers/profile-buffer.ts
@@ -336,13 +336,13 @@ export class ProfileBuffer extends BaseBuffer {
}
const chStart = performance.now();
- for (const chunk of this.chunks(toInsert, this.chunkSize)) {
- await ch.insert({
+ await this.parallelLimit(this.chunks(toInsert, this.chunkSize), (chunk) =>
+ ch.insert({
table: TABLE_NAMES.profiles,
values: chunk,
format: 'JSONEachRow',
- });
- }
+ }),
+ );
const chInsertMs = performance.now() - chStart;
const trimStart = performance.now();
diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/db/src/buffers/replay-buffer.ts
index 03112cd5c..af82b136c 100644
--- a/packages/db/src/buffers/replay-buffer.ts
+++ b/packages/db/src/buffers/replay-buffer.ts
@@ -78,13 +78,13 @@ export class ReplayBuffer extends BaseBuffer {
.filter((item): item is IClickhouseSessionReplayChunk => item != null);
const chStart = performance.now();
- for (const chunk of this.chunks(chunks, this.chunkSize)) {
- await ch.insert({
+ await this.parallelLimit(this.chunks(chunks, this.chunkSize), (chunk) =>
+ ch.insert({
table: TABLE_NAMES.session_replay_chunks,
values: chunk,
format: 'JSONEachRow',
- });
- }
+ }),
+ );
const chInsertMs = performance.now() - chStart;
const trimStart = performance.now();
diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.ts
index 2e8c6a424..24ac5e94e 100644
--- a/packages/db/src/buffers/session-buffer.ts
+++ b/packages/db/src/buffers/session-buffer.ts
@@ -14,6 +14,10 @@ export class SessionBuffer extends BaseBuffer {
? Number.parseInt(process.env.SESSION_BUFFER_CHUNK_SIZE, 10)
: 1000;
+ private readonly squashEnabled =
+ process.env.SESSION_BUFFER_SQUASH !== 'false' &&
+ process.env.SESSION_BUFFER_SQUASH !== '0';
+
private readonly redisKey = 'session-buffer';
private redis: Redis;
constructor() {
@@ -246,6 +250,80 @@ export class SessionBuffer extends BaseBuffer {
});
}
+ /**
+ * Squash multiple (sign=-1, sign=+1) pairs for the same session id into
+ * just the boundary rows: the oldest -1 (cancels the previous CH state)
+ * and the newest +1 (the final state). Single-row sessions (no update
+ * pair yet) pass through unchanged.
+ *
+ * Correctness depends on the invariant maintained by `getSession`: every
+ * update for an existing session emits exactly one -1 at version V and
+ * one +1 at V+1, where V is the existing version. The oldest -1 in the
+ * batch cancels CH's previous +1, and the newest +1 becomes the new
+ * canonical row. Intermediate (-1, +1) pairs cancel each other in CH
+ * either way — keeping them is wasted work.
+ */
+ private squashSessionsByVersion(
+ sessions: IClickhouseSession[],
+ ): IClickhouseSession[] {
+ if (sessions.length <= 1) return sessions;
+
+ const grouped = new Map<string, IClickhouseSession[]>();
+ for (const s of sessions) {
+ const key = s.id;
+ const arr = grouped.get(key);
+ if (arr) arr.push(s);
+ else grouped.set(key, [s]);
+ }
+
+ let squashedCount = 0;
+ const out: IClickhouseSession[] = [];
+
+ for (const entries of grouped.values()) {
+ if (entries.length === 1) {
+ out.push(entries[0]!);
+ continue;
+ }
+ let oldestNeg: IClickhouseSession | null = null;
+ let newestPos: IClickhouseSession | null = null;
+ for (const e of entries) {
+ if (e.sign === -1) {
+ if (!oldestNeg || e.version < oldestNeg.version) {
+ oldestNeg = e;
+ }
+ } else if (e.sign === 1) {
+ if (!newestPos || e.version > newestPos.version) {
+ newestPos = e;
+ }
+ }
+ }
+ const emitted: IClickhouseSession[] = [];
+ if (oldestNeg) emitted.push(oldestNeg);
+ if (newestPos) emitted.push(newestPos);
+ squashedCount += entries.length - emitted.length;
+ out.push(...emitted);
+ }
+
+ if (squashedCount > 0) {
+ this.logger.debug(
+ {
+ inputRows: sessions.length,
+ outputRows: out.length,
+ dropped: squashedCount,
+ uniqueSessions: grouped.size,
+ },
+ 'Session batch squashed',
+ );
+ }
+
+ console.log(
+ squashedCount > 0 ? 'Session batch squashed' : 'Session batch not squashed',
+ { inputRows: sessions.length, outputRows: out.length, dropped: squashedCount },
+ );
+
+ return out;
+ }
+
async processBuffer() {
const lrangeStart = performance.now();
const events = await this.redis.lrange(
@@ -260,23 +338,40 @@ export class SessionBuffer extends BaseBuffer {
return;
}
- const sessions = events
- .map((e) => getSafeJson<IClickhouseSession>(e))
- .map((session) => {
- return {
- ...session,
- duration: Math.max(0, session?.duration || 0),
- };
+ const parsed: IClickhouseSession[] = [];
+ for (const raw of events) {
+ const session = getSafeJson<IClickhouseSession>(raw);
+ if (!session) continue;
+ parsed.push({
+ ...session,
+ duration: Math.max(0, session.duration || 0),
});
+ }
+
+ // A single high-activity session can produce many (sign=-1, sign=+1)
+ // pairs within one flush window — each event for that session adds a
+ // pair. The VersionedCollapsingMergeTree on `sessions` will collapse
+ // them at merge time, so the final state is correct either way. But
+ // inserting all the intermediate rows costs network bytes, gzip CPU,
+ // CH ingest work, and CH merge work.
+ //
+ // We can squash within the batch and only insert the boundary rows:
+ // the oldest sign=-1 (which cancels the previous CH state) and the
+ // newest sign=+1 (the final state). Same end result, fewer rows.
+ //
+ // Set SESSION_BUFFER_SQUASH=false to disable as a safety hatch.
+ const sessions = this.squashEnabled
+ ? this.squashSessionsByVersion(parsed)
+ : parsed;
const chStart = performance.now();
- for (const chunk of this.chunks(sessions, this.chunkSize)) {
- await ch.insert({
+ await this.parallelLimit(this.chunks(sessions, this.chunkSize), (chunk) =>
+ ch.insert({
table: TABLE_NAMES.sessions,
values: chunk,
format: 'JSONEachRow',
- });
- }
+ }),
+ );
const chInsertMs = performance.now() - chStart;
const trimStart = performance.now();
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts
index 463bc31bf..0f78035f3 100644
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
@@ -112,8 +112,28 @@ function getClickhouseSettings(): ClickHouseSettings {
};
}
+// Request gzip is on by default — pays CPU on the worker to send less data
+// over the wire. For payload-heavy buffers (replay) the gzip step happens on
+// the single Node main thread and can block the event loop, so it's worth
+// being able to flip it off to test. Set CLICKHOUSE_REQUEST_COMPRESSION=false
+// to disable.
+const requestCompressionEnabled =
+ process.env.CLICKHOUSE_REQUEST_COMPRESSION !== 'false' &&
+ process.env.CLICKHOUSE_REQUEST_COMPRESSION !== '0';
+
+// Pool size per worker process. With parallel chunk inserts (cap 5 per
+// flush) and a few buffers potentially flushing concurrently across cluster
+// nodes, the previous 30 left headroom but might be tight at peak. Set
+// CLICKHOUSE_MAX_OPEN_CONNECTIONS to override.
+const maxOpenConnections = process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS
+ ? Math.max(
+ 1,
+ Number.parseInt(process.env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, 10),
+ )
+ : 50;
+
export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
- max_open_connections: 30,
+ max_open_connections: maxOpenConnections,
request_timeout: 300_000,
keep_alive: {
enabled: true,
@@ -122,7 +142,7 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {
idle_socket_ttl: 7000,
},
compression: {
- request: true,
+ request: requestCompressionEnabled,
},
clickhouse_settings: getClickhouseSettings(),
log: {
diff --git a/packages/db/src/services/session.service.ts b/packages/db/src/services/session.service.ts
index 0102999e3..4c71c3f12 100644
--- a/packages/db/src/services/session.service.ts
+++ b/packages/db/src/services/session.service.ts
@@ -50,7 +50,10 @@ export interface IClickhouseSession {
utm_content: string;
utm_term: string;
revenue: number;
- sign: 1 | 0;
+ // CollapsingMergeTree marker: +1 = current state, -1 = cancel a prior +1.
+ // The session-buffer emits both +1 (new) and -1 (old) rows per update so
+ // CH can collapse intermediate states.
+ sign: 1 | -1;
version: number;
// Dynamically added
has_replay?: boolean;