OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(ingest): profile, group and bot buffers no longer flush during ingest
Carl-Gerhard Lindesvärdc454d8ca8d93
diff --git a/packages/core/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.tsindex a9945f340..a5bb44ecb 100644--- a/packages/core/src/buffers/base-buffer.ts+++ b/packages/core/src/buffers/base-buffer.ts@@ -42,6 +42,9 @@ export interface BufferDeps {*/const DEFAULT_CH_INSERT_CONCURRENCY = 5;+/** One cron run drains as many batches as fit before the next run is due. */+const DRAIN_TIME_BUDGET_MS = 8000;+const releaseLockScript = defineRedisScript(`if redis.call("get", KEYS[1]) == ARGV[1] thenreturn redis.call("del", KEYS[1])@@ -62,6 +65,14 @@ export type FlushPhaseTimings = {onFlushMs?: number;};+const FLUSH_PHASES = [+ 'lrangeMs',+ 'chFetchMs',+ 'chInsertMs',+ 'trimMs',+ 'onFlushMs',+] as const satisfies readonly (keyof FlushPhaseTimings)[];+export type FlushTrigger = 'add' | 'cron';export type FlushObservation = {@@ -302,19 +313,43 @@ export class BaseBuffer {/*** Subclasses call this from within onFlush to record what they actually* did. The base class includes these in the FlushObservation it emits.+ * Repeated calls in one flush (one per drained batch) add up.*/protected reportFlushStats(stats: {rowsProcessed?: number;phases?: FlushPhaseTimings;}) {if (stats.rowsProcessed !== undefined) {- this.inflightStats.rowsProcessed = stats.rowsProcessed;+ this.inflightStats.rowsProcessed =+ (this.inflightStats.rowsProcessed ?? 0) + stats.rowsProcessed;}if (stats.phases) {- this.inflightStats.phases = {- ...(this.inflightStats.phases ?? {}),- ...stats.phases,- };+ const phases: FlushPhaseTimings = { ...this.inflightStats.phases };+ for (const phase of FLUSH_PHASES) {+ const ms = stats.phases[phase];+ if (ms !== undefined) {+ phases[phase] = (phases[phase] ?? 0) + ms;+ }+ }+ this.inflightStats.phases = phases;+ }+ }++ /**+ * Flushes batch after batch until one comes back short of `batchSize` or the+ * cron run's time budget is spent. Buffers only append on ingest, so the cron+ * has to keep up with a list that grows faster than one batch per run.+ */+ protected async drainBatches(+ batchSize: number,+ processBatch: () => Promise<number>+ ): Promise<void> {+ const deadline = performance.now() + DRAIN_TIME_BUDGET_MS;+ while (performance.now() < deadline) {+ const processed = await processBatch();+ if (processed < batchSize) {+ return;+ }}}diff --git a/packages/core/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.tsindex d985423a6..a53079235 100644--- a/packages/core/src/buffers/bot-buffer.ts+++ b/packages/core/src/buffers/bot-buffer.ts@@ -10,7 +10,7 @@ export class BotBuffer extends BaseBuffer {this.deps.config.buffers.bot.batchSize ?? DEFAULT_BATCH_SIZE;private readonly redisKey = 'bot-events-buffer';- private redis: Redis;+ private readonly redis: Redis;constructor(deps: BufferDeps) {super(deps, {name: 'bot',@@ -28,16 +28,7 @@ export class BotBuffer extends BaseBuffer {async add(event: IClickhouseBotEvent) {return this.timeAdd(async () => {try {- const result = await this.redis- .multi()- .rpush(this.redisKey, JSON.stringify(event))- .llen(this.redisKey)- .exec();-- const bufferLength = (result?.[1]?.[1] as number) ?? 0;- if (bufferLength >= this.batchSize) {- await this.tryFlush({ trigger: 'add' });- }+ await this.redis.rpush(this.redisKey, JSON.stringify(event));} catch (error) {this.logger.error({ err: error }, 'Failed to add bot event');}@@ -45,6 +36,10 @@ export class BotBuffer extends BaseBuffer {}async processBuffer() {+ await this.drainBatches(this.batchSize, () => this.processBatch());+ }++ private async processBatch(): Promise<number> {const lrangeStart = performance.now();const events = await this.redis.lrange(this.redisKey,@@ -55,7 +50,7 @@ export class BotBuffer extends BaseBuffer {if (events.length === 0) {this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });- return;+ return 0;}// Raw passthrough: each Redis entry is already a JSONEachRow line.@@ -77,5 +72,6 @@ export class BotBuffer extends BaseBuffer {rowsProcessed: events.length,phases: { lrangeMs, chInsertMs, trimMs },});+ return events.length;}}diff --git a/packages/core/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.tsindex 024195141..50845ca66 100644--- a/packages/core/src/buffers/group-buffer.ts+++ b/packages/core/src/buffers/group-buffer.ts@@ -49,7 +49,7 @@ export class GroupBuffer extends BaseBuffer {private readonly redisKey = 'group-buffer';private readonly redisCachePrefix = 'group-cache:';- private redis: Redis;+ private readonly redis: Redis;constructor(deps: BufferDeps) {super(deps, {@@ -140,17 +140,10 @@ export class GroupBuffer extends BaseBuffer {.multi().set(cacheKey, JSON.stringify(cacheEntry), 'EX', this.ttlInSeconds).rpush(this.redisKey, JSON.stringify(entry))- .llen(this.redisKey).exec();if (!result) {this.logger.error({ input }, 'Failed to add group to Redis');- return;- }-- const bufferLength = (result?.[2]?.[1] as number) ?? 0;- if (bufferLength >= this.batchSize) {- await this.tryFlush({ trigger: 'add' });}} catch (error) {this.logger.error({ err: error, input }, 'Failed to add group');@@ -163,13 +156,17 @@ export class GroupBuffer extends BaseBuffer {}async processBuffer(): Promise<void> {+ await this.drainBatches(this.batchSize, () => this.processBatch());+ }++ private async processBatch(): Promise<number> {const lrangeStart = performance.now();const items = await this.redis.lrange(this.redisKey, 0, this.batchSize - 1);const lrangeMs = performance.now() - lrangeStart;if (items.length === 0) {this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });- return;+ return 0;}// Raw passthrough: each Redis entry is already a JSONEachRow line.@@ -193,5 +190,6 @@ export class GroupBuffer extends BaseBuffer {rowsProcessed: items.length,phases: { lrangeMs, chInsertMs, trimMs },});+ return items.length;}}diff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.tsindex b4910488d..6784083e3 100644--- a/packages/core/src/buffers/profile-buffer.test.ts+++ b/packages/core/src/buffers/profile-buffer.test.ts@@ -1,6 +1,7 @@import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';import { getRedisCache } from '@openpanel/redis';import { bufferDepsWithCh } from '../../test/buffer-fixtures';+import { testCoreConfig } from '../../test/config-fixture';import type { IClickhouseProfile } from '../modules/profile/profile.service';// The client comes in as `BufferDeps.ch` and reads go through core's own@@ -80,6 +81,24 @@ describe('ProfileBuffer', () => {expect(sizeAfter).toBe(sizeBefore + 1);});+ it('add never flushes; a cron flush drains past one batch', async () => {+ const config = testCoreConfig();+ config.buffers.profile.batchSize = 2;+ const buffer = new ProfileBuffer(+ bufferDepsWithCh({ insert: chInsert }, config)+ );++ for (let i = 0; i < 5; i++) {+ await buffer.add(makeProfile({ id: `profile-${i}` }));+ }+ expect(chInsert).not.toHaveBeenCalled();++ await buffer.tryFlush({ trigger: 'cron' });++ expect(chInsert).toHaveBeenCalledTimes(3);+ expect(await buffer.getBufferSize()).toBe(0);+ });+it('concurrent adds: both raw profiles are queued', async () => {const identifyProfile = makeProfile({first_name: 'John',diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.tsindex d4f447d8b..3660cf2a3 100644--- a/packages/core/src/buffers/profile-buffer.ts+++ b/packages/core/src/buffers/profile-buffer.ts@@ -119,21 +119,7 @@ export class ProfileBuffer extends BaseBuffer {}}- const result = await this.redis- .multi()- .rpush(this.redisKey, JSON.stringify(profile))- .llen(this.redisKey)- .exec();-- if (!result) {- this.logger.error({ profile }, 'Failed to add profile to Redis');- return;- }-- const bufferLength = (result?.[1]?.[1] as number) ?? 0;- if (bufferLength >= this.batchSize) {- await this.tryFlush({ trigger: 'add' });- }+ await this.redis.rpush(this.redisKey, JSON.stringify(profile));} catch (error) {this.logger.error({ err: error, profile }, 'Failed to add profile');}@@ -241,6 +227,10 @@ export class ProfileBuffer extends BaseBuffer {}async processBuffer() {+ await this.drainBatches(this.batchSize, () => this.processBatch());+ }++ private async processBatch(): Promise<number> {const lrangeStart = performance.now();const rawProfiles = await this.redis.lrange(this.redisKey,@@ -251,7 +241,7 @@ export class ProfileBuffer extends BaseBuffer {if (rawProfiles.length === 0) {this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });- return;+ return 0;}// Yields to the event loop periodically so concurrent add() calls and@@ -367,5 +357,6 @@ export class ProfileBuffer extends BaseBuffer {rowsProcessed: rawProfiles.length,phases: { lrangeMs, chFetchMs, chInsertMs, trimMs },});+ return rawProfiles.length;}}diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tsindex c3b94f74e..dc097c9f7 100644--- a/packages/core/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -81,8 +81,6 @@ function pickUtm(const DEFAULT_BATCH_SIZE = 1000;const DEFAULT_CHUNK_SIZE = 1000;-/** One cron run drains as many batches as fit before the next run is due. */-const DRAIN_TIME_BUDGET_MS = 8000;export class SessionBuffer extends BaseBuffer {private readonly batchSize =@@ -501,18 +499,7 @@ export class SessionBuffer extends BaseBuffer {}async processBuffer() {- const deadline = performance.now() + DRAIN_TIME_BUDGET_MS;- let rowsProcessed = 0;-- while (performance.now() < deadline) {- const processed = await this.processBatch();- rowsProcessed += processed;- if (processed < this.batchSize) {- break;- }- }-- this.reportFlushStats({ rowsProcessed });+ await this.drainBatches(this.batchSize, () => this.processBatch());}/** Inserts and trims one batch from the head of the list; returns its size. */@@ -561,6 +548,7 @@ export class SessionBuffer extends BaseBuffer {// (and the next flush retries the still-queued rows).await this.redis.ltrim(this.redisKey, events.length, -1);+ this.reportFlushStats({ rowsProcessed: events.length });this.logger.debug({ count: events.length }, 'Processed sessions');return events.length;}diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex c0a4d3bbc..a1ddc3bb7 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -7,6 +7,7 @@ const GOLDEN_SCHEDULER_IDS = ['cohortRefresh','dataHealth','delete',+ 'flushBots','flushEvents','flushExports','flushGroups',@@ -108,6 +109,7 @@ test('the cron queue carries the insight module cron fragment', () => {'cohortRefresh','dataHealth','delete',+ 'flushBots','flushExports','flushEvents','flushGroups',@@ -221,7 +223,7 @@ test('the notification queue carries the notification module job', () => {});});-test('the cron queue derives the golden 20 scheduler ids', () => {+test('the cron queue derives the golden 21 scheduler ids', () => {const derivedIds = [...schedulersFromRegistry(queues.cron).map((s) => s.id),PING_SCHEDULE.id,diff --git a/packages/core/src/jobs/schedulers.test.ts b/packages/core/src/jobs/schedulers.test.tsindex 3a70c983b..c66fc724b 100644--- a/packages/core/src/jobs/schedulers.test.ts+++ b/packages/core/src/jobs/schedulers.test.ts@@ -7,12 +7,13 @@ import {startSchedulers,} from './schedulers';-// Byte-identity snapshot: these 20 scheduler ids must not change without a+// Byte-identity snapshot: these 21 scheduler ids must not change without a// migration, since BullMQ persists them as job-scheduler keys in Redis.const GOLDEN_SCHEDULER_IDS = ['cohortRefresh','dataHealth','delete',+ 'flushBots','flushEvents','flushExports','flushGroups',@@ -99,7 +100,7 @@ function fakeQueue(options?: {test('the golden scheduler ids are reproduced byte-for-byte', () => {const allIds = [...CRON_SCHEDULES.map((s) => s.id), PING_SCHEDULE.id].sort();- expect(allIds).toHaveLength(20);+ expect(allIds).toHaveLength(21);expect(allIds).toEqual(GOLDEN_SCHEDULER_IDS);});@@ -121,7 +122,7 @@ test('ping is excluded by default — neither self-hosted nor production', async});const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();- expect(upsertedIds).toHaveLength(19);+ expect(upsertedIds).toHaveLength(20);expect(upsertedIds).not.toContain('ping');});@@ -162,7 +163,7 @@ test('ping is included only when self-hosted AND production', async () => {});const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();- expect(upsertedIds).toHaveLength(20);+ expect(upsertedIds).toHaveLength(21);expect(upsertedIds).toEqual(GOLDEN_SCHEDULER_IDS);});diff --git a/packages/core/src/modules/event/event.jobs.ts b/packages/core/src/modules/event/event.jobs.tsindex 397cfcde0..3882e3e70 100644--- a/packages/core/src/modules/event/event.jobs.ts+++ b/packages/core/src/modules/event/event.jobs.ts@@ -1,9 +1,11 @@-// The event buffer (Redis -> ClickHouse `events`) is this module's domain.+// The event buffer (Redis -> ClickHouse `events`) and the bot buffer+// (-> `events_bots`) are this module's domain.import { z } from 'zod';import { defineJob } from '../../jobs/define';const FLUSH_EVENTS_INTERVAL_MS = 10_000;+const FLUSH_BOTS_INTERVAL_MS = 30_000;/** This module's fragment of the `cron` queue's jobs. */export const eventCronJobs = {@@ -14,4 +16,11 @@ export const eventCronJobs = {await ctx.buffers.event.tryFlush({ trigger: 'cron' });},}),+ flushBots: defineJob({+ payload: z.null(),+ cron: { every: FLUSH_BOTS_INTERVAL_MS },+ handler: async ({ ctx }) => {+ await ctx.buffers.bot.tryFlush({ trigger: 'cron' });+ },+ }),};