OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: avoid json parse data from redis in buffers
Carl-Gerhard Lindesvärddfd941bf5886
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex 523e2f3b1..ff8634890 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -1,3 +1,4 @@+import { Readable } from 'node:stream';import type { ClickHouseSettings } from '@clickhouse/client';import { generateSecureId } from '@openpanel/common/server';import { createLogger, type ILogger } from '@openpanel/logger';@@ -132,6 +133,60 @@ export class BaseBuffer {return new Promise((resolve) => setImmediate(resolve));}+ /**+ * Compute how often to yield in a CPU-bound loop over `batchSize`+ * items: target ~20 yields per flush, clamped to a per-buffer band.+ *+ * The band lets each buffer reflect per-item cost:+ * - cheap loops (regex, shallow checks): min=1000, max=5000 → coarse+ * - heavy loops (JSON.parse + deep merge): min=100, max=1000 → fine+ *+ * Without clamping, small default batchSizes (e.g. profile-buffer's+ * 200) would yield far too often (every ~10 items, dominated by yield+ * overhead) or — with hardcoded `% N` literals where N > batchSize —+ * never yield at all. Both are bad; the band rules out both.+ *+ * @param batchSize total items the loop will process+ * @param opts.min lower bound for yield interval (default 100)+ * @param opts.max upper bound for yield interval (default 5000)+ * @param opts.targetYieldsPerFlush default 20+ */+ protected getYieldInterval(+ batchSize: number,+ opts: { min?: number; max?: number; targetYieldsPerFlush?: number } = {},+ ): number {+ const target = opts.targetYieldsPerFlush ?? 20;+ const min = opts.min ?? 100;+ const max = opts.max ?? 5000;+ return Math.max(min, Math.min(max, Math.ceil(batchSize / target)));+ }++ /**+ * Wrap an array of already-serialized JSONEachRow lines into an+ * object-mode Readable stream that yields the lines as strings. The+ * @clickhouse/client requires object-mode for JSON* formats — but with+ * our custom `json.stringify` in `CLICKHOUSE_OPTIONS` (passes strings+ * through unchanged), the lines reach CH without being re-serialized.+ *+ * Net effect: no JSON.parse on our side, no JSON.stringify on the+ * client's side — the bytes go from Redis to CH's HTTP body+ * essentially unchanged.+ *+ * Caller must guarantee each line is a valid single-line JSON object+ * (the contract when we JSON.stringify(obj) before rpush'ing into+ * Redis). A malformed line will fail the whole CH insert, not just+ * that row.+ */+ protected jsonEachRowStream(lines: string[]): Readable {+ return Readable.from(+ (function* () {+ for (const line of lines) {+ yield line;+ }+ })(),+ );+ }+protected chunks<T>(items: T[], size: number) {const chunks = [];for (let i = 0; i < items.length; i += size) {diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.tsindex 7c645a9aa..3224647c6 100644--- a/packages/db/src/buffers/bot-buffer.ts+++ b/packages/db/src/buffers/bot-buffer.ts@@ -1,4 +1,3 @@-import { getSafeJson } from '@openpanel/json';import { getRedisCache, type Redis } from '@openpanel/redis';import { ch, TABLE_NAMES } from '../clickhouse/client';import type { IClickhouseBotEvent } from '../services/event.service';@@ -58,12 +57,13 @@ export class BotBuffer extends BaseBuffer {return;}- const parsedEvents = events.map((e) => getSafeJson<IClickhouseBotEvent>(e));-+ // Raw passthrough: each Redis entry is already a valid JSONEachRow+ // line. Streaming raw strings skips JSON.parse + the client's+ // re-stringify on the hot path.const chStart = performance.now();await ch.insert({table: TABLE_NAMES.events_bots,- values: parsedEvents,+ values: this.jsonEachRowStream(events),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),});diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/db/src/buffers/event-buffer.test.tsindex 3e82aa6cb..66e95f130 100644--- a/packages/db/src/buffers/event-buffer.test.ts+++ b/packages/db/src/buffers/event-buffer.test.ts@@ -1,3 +1,4 @@+import type { Readable } from 'node:stream';import { getRedisCache } from '@openpanel/redis';import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';import * as chClient from '../clickhouse/client';@@ -6,7 +7,19 @@ const { ch } = chClient;// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffervi.mock('../services/event.service', () => ({}));-import { EventBuffer } from './event-buffer';+import { EventBuffer, extractProjectId } from './event-buffer';++/** Drain an object-mode Readable into an array of line strings. event-buffer+ * yields each JSONEachRow row as its own string chunk; the @clickhouse/client+ * adds the trailing '\n' itself via encodeJSON. So in production each `\n` is+ * added by the client, but here we just collect chunks 1:1. */+async function streamToLines(stream: Readable): Promise<string[]> {+ const lines: string[] = [];+ for await (const chunk of stream) {+ lines.push(typeof chunk === 'string' ? chunk : chunk.toString('utf8'));+ }+ return lines;+}const redis = getRedisCache();@@ -171,7 +184,21 @@ describe('EventBuffer', () => {const callArgs = insertSpy.mock.calls[0]![0];expect(callArgs.format).toBe('JSONEachRow');expect(callArgs.table).toBe('events');- expect(Array.isArray(callArgs.values)).toBe(true);+ // After the raw-passthrough optimisation, `values` is an+ // object-mode Readable stream that yields pre-serialized+ // JSONEachRow lines as strings. The client (configured with+ // `json.stringify: (v) => typeof v === 'string' ? v : JSON.stringify(v)`)+ // then passes the strings through unchanged — no JSON.parse on+ // our side, no JSON.stringify on the client's side. Object mode+ // is mandatory: @clickhouse/client rejects byte streams for+ // JSON* formats with "expected Readable Stream with enabled+ // object mode".+ const stream = callArgs.values as Readable;+ expect(stream.readableObjectMode).toBe(true);+ const lines = await streamToLines(stream);+ expect(lines.length).toBe(2);+ expect(JSON.parse(lines[0]!).name).toBe('event1');+ expect(JSON.parse(lines[1]!).name).toBe('event2');expect(await eventBuffer.getBufferSize()).toBe(0);@@ -199,10 +226,14 @@ describe('EventBuffer', () => {await eb.processBuffer();expect(insertSpy).toHaveBeenCalledTimes(2);- const call1Values = insertSpy.mock.calls[0]![0].values as any[];- const call2Values = insertSpy.mock.calls[1]![0].values as any[];- expect(call1Values.length).toBe(2);- expect(call2Values.length).toBe(2);+ const call1 = await streamToLines(+ insertSpy.mock.calls[0]![0].values as Readable,+ );+ const call2 = await streamToLines(+ insertSpy.mock.calls[1]![0].values as Readable,+ );+ expect(call1.length).toBe(2);+ expect(call2.length).toBe(2);if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;@@ -296,3 +327,77 @@ describe('EventBuffer', () => {insertSpy.mockRestore();});});++describe('extractProjectId', () => {+ it('extracts the top-level project_id', () => {+ const line = JSON.stringify({+ id: 'evt1',+ name: 'foo',+ project_id: 'real-project',+ });+ expect(extractProjectId(line)).toBe('real-project');+ });++ it('returns null when the field is absent', () => {+ const line = JSON.stringify({ id: 'evt1', name: 'foo' });+ expect(extractProjectId(line)).toBeNull();+ });++ it('returns null on malformed input', () => {+ expect(extractProjectId('not json at all')).toBeNull();+ expect(extractProjectId('')).toBeNull();+ });++ it('returns null when project_id is empty', () => {+ // Empty string never matches `"project_id":"..."` — the value side+ // is at least one char, so this drops out cleanly.+ const line = JSON.stringify({ project_id: '' });+ expect(extractProjectId(line)).toBeNull();+ });++ it('falls back to JSON.parse when properties also has a project_id (after)', () => {+ // Top-level project_id is serialized first (event.service.ts order).+ // Fast path would still match it correctly here, but the helper+ // detects the second occurrence and routes to JSON.parse for safety.+ const line = JSON.stringify({+ id: 'evt1',+ project_id: 'real-project',+ properties: { project_id: 'user-supplied' },+ });+ expect(extractProjectId(line)).toBe('real-project');+ });++ it('falls back to JSON.parse when properties has project_id BEFORE top-level', () => {+ // This is the bug case the regex couldn't handle. If a future+ // refactor swaps the field order in the event constructor, the+ // fast path would attribute counts to the wrong project. The+ // fallback fixes that: as soon as we see two occurrences we+ // resolve via real JSON.parse and pick the top-level key.+ const line = JSON.stringify({+ properties: { project_id: 'user-supplied' },+ project_id: 'real-project',+ });+ expect(extractProjectId(line)).toBe('real-project');+ });++ it('does not confuse escaped quotes in a string value with the real field', () => {+ // A user types `"project_id":"x"` inside a property value. After+ // JSON.stringify, those quotes are escaped (`\"`), so the+ // indexOf scan for `"project_id":"` (no backslash) can't match+ // them. Only the real top-level field is found.+ const line = JSON.stringify({+ id: 'evt1',+ project_id: 'real-project',+ properties: { note: 'has "project_id":"fake" inside' },+ });+ expect(extractProjectId(line)).toBe('real-project');+ });++ it('handles a project_id nested deep inside properties (fallback path)', () => {+ const line = JSON.stringify({+ project_id: 'real-project',+ properties: { nested: { project_id: 'deep-fake' } },+ });+ expect(extractProjectId(line)).toBe('real-project');+ });+});diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 8073e3097..d29003f63 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -1,9 +1,52 @@-import { getSafeJson } from '@openpanel/json';import { getRedisCache, publishEvent } from '@openpanel/redis';import { ch, chQuery } from '../clickhouse/client';import type { IClickhouseEvent } from '../services/event.service';import { BaseBuffer } from './base-buffer';+const PROJECT_ID_NEEDLE = '"project_id":"';++/**+ * Extract the top-level `project_id` from a single JSONEachRow event+ * line without doing a full JSON.parse.+ *+ * Fast path: scan with `indexOf`. If the needle appears exactly once,+ * we know it's the top-level field (any `project_id` nested in a JSON+ * string value would be escaped as `\"project_id\":\"...`, which the+ * indexOf scan can't match because of the `\` in front).+ *+ * Slow path: if the needle appears two or more times, the line has a+ * legitimate nested `project_id` key (e.g. inside `properties` if a+ * user happens to set one with that name). The regex/indexOf can't+ * tell which is the top-level one — at that point we fall back to a+ * real JSON.parse for correctness. This is rare in practice but+ * removes the silent-attribution failure mode entirely.+ *+ * Returns `null` if no top-level project_id is present or the row is+ * malformed.+ */+export function extractProjectId(line: string): string | null {+ const first = line.indexOf(PROJECT_ID_NEEDLE);+ if (first < 0) return null;++ const second = line.indexOf(PROJECT_ID_NEEDLE, first + PROJECT_ID_NEEDLE.length);+ if (second >= 0) {+ try {+ const obj = JSON.parse(line) as { project_id?: unknown };+ return typeof obj.project_id === 'string' ? obj.project_id : null;+ } catch {+ return null;+ }+ }++ const valueStart = first + PROJECT_ID_NEEDLE.length;+ const valueEnd = line.indexOf('"', valueStart);+ // valueEnd === valueStart means the value is empty (`"project_id":""`)+ // — treat as missing so we don't pollute pub/sub counts with an empty+ // key. Matches the old regex's `[^"]+` (one-or-more) behavior.+ if (valueEnd <= valueStart) return null;+ return line.slice(valueStart, valueEnd);+}+export class EventBuffer extends BaseBuffer {private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)@@ -146,49 +189,54 @@ export class EventBuffer extends BaseBuffer {return;}- // Parse with periodic yields. Event batches can be very large- // (EVENT_BUFFER_BATCH_SIZE defaults to 4000 but is often set to- // 100k in prod). Yielding every 1000 items keeps the event loop- // responsive without measurable throughput impact.- const eventsToClickhouse: IClickhouseEvent[] = [];+ // We don't need to JSON.parse the events at all — they're already+ // valid JSONEachRow lines (one stringified event per Redis entry).+ // The client's custom `json.stringify` (set in CLICKHOUSE_OPTIONS)+ // passes strings through unchanged, so the bytes go straight from+ // Redis → CH HTTP body. This skips:+ // - JSON.parse × N (50–300ms for N=100k)+ // - The @clickhouse/client's internal JSON.stringify × N (same)+ // - All the intermediate object allocations (saves ~200MB heap)+ //+ // We still need `project_id` per row for the per-project pub/sub.+ // extractProjectId() does an indexOf-based fast path that's ~50×+ // faster than JSON.parse, and falls back to a real parse on the+ // rare line where `project_id` appears more than once (e.g. a+ // user-supplied `properties.project_id`) — so the count is always+ // attributed to the top-level field, never a nested one.+ const countByProject = new Map<string, number>();+ const yieldEvery = this.getYieldInterval(queueEvents.length, {+ min: 1000,+ max: 5000,+ });for (let i = 0; i < queueEvents.length; i++) {- const event = getSafeJson<IClickhouseEvent>(queueEvents[i]!);- if (event) {- if (!Array.isArray(event.groups)) {- event.groups = [];- }- eventsToClickhouse.push(event);+ const projectId = extractProjectId(queueEvents[i]!);+ if (projectId) {+ countByProject.set(+ projectId,+ (countByProject.get(projectId) ?? 0) + 1,+ );}- if ((i + 1) % 1000 === 0) {+ if ((i + 1) % yieldEvery === 0) {await this.yieldToEventLoop();}}- if (eventsToClickhouse.length === 0) {- this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });- return;- }-const chStart = performance.now();await this.parallelLimit(- this.chunks(eventsToClickhouse, this.chunkSize),+ this.chunks(queueEvents, this.chunkSize),(chunk) =>ch.insert({table: 'events',- values: chunk,+ // Stream the raw JSONEachRow lines straight through — already+ // serialized in Redis, no client-side parse/stringify needed.+ values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- })+ }),);const chInsertMs = performance.now() - chStart;- const countByProject = new Map<string, number>();- for (const event of eventsToClickhouse) {- countByProject.set(- event.project_id,- (countByProject.get(event.project_id) ?? 0) + 1- );- }for (const [projectId, count] of countByProject) {publishEvent('events', 'batch', { projectId, count });}diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/db/src/buffers/group-buffer.tsindex d42e08f79..feb66fcfd 100644--- a/packages/db/src/buffers/group-buffer.ts+++ b/packages/db/src/buffers/group-buffer.ts@@ -176,16 +176,17 @@ export class GroupBuffer extends BaseBuffer {return;}- const parsed = items.map((i) => getSafeJson<IGroupBufferEntry>(i));-+ // Raw passthrough: each Redis entry is already a valid JSONEachRow+ // line. Streaming raw strings to CH skips JSON.parse + the client's+ // re-stringify on the hot path.const chStart = performance.now();- await this.parallelLimit(this.chunks(parsed, this.chunkSize), (chunk) =>+ await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>ch.insert({table: TABLE_NAMES.groups,- values: chunk,+ values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- })+ }),);const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex a9fc0297c..554467d4c 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -260,11 +260,19 @@ export class ProfileBuffer extends BaseBuffer {}// Parse + merge within batch in a single pass. Yields back to the- // event loop every 500 items so concurrent add() calls and BullMQ- // heartbeats can make progress while the merge churns through 10k++ // event loop periodically so concurrent add() calls and BullMQ+ // heartbeats can make progress while the merge churns through// profiles. The merge step is the hottest CPU stage of any buffer —// deepMergeObjects recursively combines nested `properties` objects// and was previously responsible for ~500ms event-loop blocks.+ //+ // Heavy loop (parse + deep merge): narrow band so we yield often+ // even with smaller batches but don't yield every-few-items on+ // very small ones.+ const mergeYieldEvery = this.getYieldInterval(rawProfiles.length, {+ min: 100,+ max: 1000,+ });const mergedInBatch = new Map<string, IClickhouseProfile>();for (let i = 0; i < rawProfiles.length; i++) {const profile = getSafeJson<IClickhouseProfile>(rawProfiles[i]!);@@ -274,7 +282,7 @@ export class ProfileBuffer extends BaseBuffer {key,this.mergeProfiles(mergedInBatch.get(key) ?? null, profile),);- if ((i + 1) % 500 === 0) {+ if ((i + 1) % mergeYieldEvery === 0) {await this.yieldToEventLoop();}}@@ -320,8 +328,12 @@ export class ProfileBuffer extends BaseBuffer {}// Final merge: in-batch profile + existing (from cache or ClickHouse).- // Second heavy stage — same deepMergeObjects call per unique profile,- // so the same yield-every-500 pattern applies.+ // Second heavy stage — same deepMergeObjects call per unique profile.+ // Same band as the in-batch merge above.+ const finalYieldEvery = this.getYieldInterval(uniqueProfiles.length, {+ min: 100,+ max: 1000,+ });const toInsert: IClickhouseProfile[] = [];const multi = this.redis.multi();@@ -342,7 +354,7 @@ export class ProfileBuffer extends BaseBuffer {'EX',this.ttlInSeconds,);- if ((i + 1) % 500 === 0) {+ if ((i + 1) % finalYieldEvery === 0) {await this.yieldToEventLoop();}}diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/db/src/buffers/replay-buffer.tsindex cf7569ba9..b2c1cefa2 100644--- a/packages/db/src/buffers/replay-buffer.ts+++ b/packages/db/src/buffers/replay-buffer.ts@@ -1,4 +1,3 @@-import { getSafeJson } from '@openpanel/json';import { getRedisCache } from '@openpanel/redis';import { ch, TABLE_NAMES } from '../clickhouse/client';import { BaseBuffer } from './base-buffer';@@ -72,27 +71,19 @@ export class ReplayBuffer extends BaseBuffer {return;}- // Parse rrweb chunks with periodic yields. Each chunk's `payload`- // field is a large JSON blob (~10-100KB) so JSON.parse is expensive- // per item. Yield every 100 items to keep the event loop responsive- // during big flushes.- const chunks: IClickhouseSessionReplayChunk[] = [];- for (let i = 0; i < items.length; i++) {- const parsed = getSafeJson<IClickhouseSessionReplayChunk>(items[i]!);- if (parsed != null) chunks.push(parsed);- if ((i + 1) % 100 === 0) {- await this.yieldToEventLoop();- }- }-+ // Raw passthrough: each Redis entry is already a valid JSONEachRow+ // line (we JSON.stringify a single chunk before rpush). Streaming the+ // raw strings to CH skips JSON.parse × N on the worker AND the+ // client's internal JSON.stringify × N — significant because each+ // rrweb chunk's `payload` is 10–100KB.const chStart = performance.now();- await this.parallelLimit(this.chunks(chunks, this.chunkSize), (chunk) =>+ await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>ch.insert({table: TABLE_NAMES.session_replay_chunks,- values: chunk,+ values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- })+ }),);const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 1277f6d61..fc4941af8 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -10,7 +10,7 @@ import type { IInterval } from '@openpanel/validation';import sqlstring from 'sqlstring';import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';-export { createClient };+export { createClient } from '@clickhouse/client';const logger = createLogger({ name: 'clickhouse' });@@ -171,6 +171,16 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {LoggerClass: CustomLogger,level: ClickHouseLogLevel.DEBUG,},+ // Custom JSON serializer used on inserts. For buffers that already+ // hold JSONEachRow lines as strings (event/replay/bot/group — pulled+ // straight out of Redis), this is a passthrough — no JSON.stringify+ // on the hot path. For buffers that pass real objects (session /+ // profile, which need to parse + transform before inserting), it+ // falls back to JSON.stringify. The client appends '\n' itself.+ json: {+ stringify: <T>(value: T): string =>+ typeof value === 'string' ? value : JSON.stringify(value),+ },};// CLICKHOUSE_URL accepts a single URL or comma-separated list. With a list,@@ -197,7 +207,9 @@ const clients: ClickHouseClient[] = clickhouseUrls.map((url) =>const picker = new RoundRobinPicker(clients, clickhouseUrls, unhealthyMarkMs);function maskUrlCredentials(url: string): string {- if (!url) return '<unset>';+ if (!url) {+ return '<unset>';+ }try {const u = new URL(url);if (u.username) {@@ -246,15 +258,17 @@ const cleanQuery = (query?: string) =>export async function withRetry<T>(operation: (client: ClickHouseClient,- ctx: { url: string; index: number },- ) => Promise<T>,+ ctx: { url: string; index: number }+ ) => Promise<T>): Promise<T> {return withRoundRobinRetry(picker, operation, logger);}/** Best-effort URL → hostname extraction for log labels. */function urlHostname(url: string | undefined): string | undefined {- if (!url) return undefined;+ if (!url) {+ return undefined;+ }try {return new URL(url).hostname;} catch {diff --git a/packages/validation/src/track.validation.ts b/packages/validation/src/track.validation.tsindex c97b51a08..eae1f055e 100644--- a/packages/validation/src/track.validation.ts+++ b/packages/validation/src/track.validation.ts@@ -31,7 +31,7 @@ export interface ITrackPayload {name: string;properties?: Record<string, unknown>;profileId?: IProfileId;- groups?: string[];+ groups?: string[] | null;}export interface IIdentifyPayload {@@ -103,7 +103,7 @@ export const zTrackPayload = zname: z.string().min(1),properties: z.record(z.string(), z.unknown()).optional(),profileId: zProfileId.optional(),- groups: z.array(z.string().min(1)).optional(),+ groups: z.array(z.string().min(1)).nullish(),}).refine((data) => !RESERVED_EVENT_NAMES.includes(data.name as any), {message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,