Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(ingest): profile, group and bot buffers no longer flush during ingest

Carl-Gerhard Lindesvärdc454d8ca8d93
diff --git a/packages/core/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.ts
index a9945f340..a5bb44ecb 100644
--- a/packages/core/src/buffers/base-buffer.ts
+++ b/packages/core/src/buffers/base-buffer.ts
@@ -42,6 +42,9 @@ export interface BufferDeps {
*/
const DEFAULT_CH_INSERT_CONCURRENCY = 5;
+/** One cron run drains as many batches as fit before the next run is due. */
+const DRAIN_TIME_BUDGET_MS = 8000;
+
const releaseLockScript = defineRedisScript(`
if redis.call("get", KEYS[1]) == ARGV[1] then
return redis.call("del", KEYS[1])
@@ -62,6 +65,14 @@ export type FlushPhaseTimings = {
onFlushMs?: number;
};
+const FLUSH_PHASES = [
+ 'lrangeMs',
+ 'chFetchMs',
+ 'chInsertMs',
+ 'trimMs',
+ 'onFlushMs',
+] as const satisfies readonly (keyof FlushPhaseTimings)[];
+
export type FlushTrigger = 'add' | 'cron';
export type FlushObservation = {
@@ -302,19 +313,43 @@ export class BaseBuffer {
/**
* Subclasses call this from within onFlush to record what they actually
* did. The base class includes these in the FlushObservation it emits.
+ * Repeated calls in one flush (one per drained batch) add up.
*/
protected reportFlushStats(stats: {
rowsProcessed?: number;
phases?: FlushPhaseTimings;
}) {
if (stats.rowsProcessed !== undefined) {
- this.inflightStats.rowsProcessed = stats.rowsProcessed;
+ this.inflightStats.rowsProcessed =
+ (this.inflightStats.rowsProcessed ?? 0) + stats.rowsProcessed;
}
if (stats.phases) {
- this.inflightStats.phases = {
- ...(this.inflightStats.phases ?? {}),
- ...stats.phases,
- };
+ const phases: FlushPhaseTimings = { ...this.inflightStats.phases };
+ for (const phase of FLUSH_PHASES) {
+ const ms = stats.phases[phase];
+ if (ms !== undefined) {
+ phases[phase] = (phases[phase] ?? 0) + ms;
+ }
+ }
+ this.inflightStats.phases = phases;
+ }
+ }
+
+ /**
+ * Flushes batch after batch until one comes back short of `batchSize` or the
+ * cron run's time budget is spent. Buffers only append on ingest, so the cron
+ * has to keep up with a list that grows faster than one batch per run.
+ */
+ protected async drainBatches(
+ batchSize: number,
+ processBatch: () => Promise<number>
+ ): Promise<void> {
+ const deadline = performance.now() + DRAIN_TIME_BUDGET_MS;
+ while (performance.now() < deadline) {
+ const processed = await processBatch();
+ if (processed < batchSize) {
+ return;
+ }
}
}
diff --git a/packages/core/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.ts
index d985423a6..a53079235 100644
--- a/packages/core/src/buffers/bot-buffer.ts
+++ b/packages/core/src/buffers/bot-buffer.ts
@@ -10,7 +10,7 @@ export class BotBuffer extends BaseBuffer {
this.deps.config.buffers.bot.batchSize ?? DEFAULT_BATCH_SIZE;
private readonly redisKey = 'bot-events-buffer';
- private redis: Redis;
+ private readonly redis: Redis;
constructor(deps: BufferDeps) {
super(deps, {
name: 'bot',
@@ -28,16 +28,7 @@ export class BotBuffer extends BaseBuffer {
async add(event: IClickhouseBotEvent) {
return this.timeAdd(async () => {
try {
- const result = await this.redis
- .multi()
- .rpush(this.redisKey, JSON.stringify(event))
- .llen(this.redisKey)
- .exec();
-
- const bufferLength = (result?.[1]?.[1] as number) ?? 0;
- if (bufferLength >= this.batchSize) {
- await this.tryFlush({ trigger: 'add' });
- }
+ await this.redis.rpush(this.redisKey, JSON.stringify(event));
} catch (error) {
this.logger.error({ err: error }, 'Failed to add bot event');
}
@@ -45,6 +36,10 @@ export class BotBuffer extends BaseBuffer {
}
async processBuffer() {
+ await this.drainBatches(this.batchSize, () => this.processBatch());
+ }
+
+ private async processBatch(): Promise<number> {
const lrangeStart = performance.now();
const events = await this.redis.lrange(
this.redisKey,
@@ -55,7 +50,7 @@ export class BotBuffer extends BaseBuffer {
if (events.length === 0) {
this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
- return;
+ return 0;
}
// Raw passthrough: each Redis entry is already a JSONEachRow line.
@@ -77,5 +72,6 @@ export class BotBuffer extends BaseBuffer {
rowsProcessed: events.length,
phases: { lrangeMs, chInsertMs, trimMs },
});
+ return events.length;
}
}
diff --git a/packages/core/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.ts
index 024195141..50845ca66 100644
--- a/packages/core/src/buffers/group-buffer.ts
+++ b/packages/core/src/buffers/group-buffer.ts
@@ -49,7 +49,7 @@ export class GroupBuffer extends BaseBuffer {
private readonly redisKey = 'group-buffer';
private readonly redisCachePrefix = 'group-cache:';
- private redis: Redis;
+ private readonly redis: Redis;
constructor(deps: BufferDeps) {
super(deps, {
@@ -140,17 +140,10 @@ export class GroupBuffer extends BaseBuffer {
.multi()
.set(cacheKey, JSON.stringify(cacheEntry), 'EX', this.ttlInSeconds)
.rpush(this.redisKey, JSON.stringify(entry))
- .llen(this.redisKey)
.exec();
if (!result) {
this.logger.error({ input }, 'Failed to add group to Redis');
- return;
- }
-
- const bufferLength = (result?.[2]?.[1] as number) ?? 0;
- if (bufferLength >= this.batchSize) {
- await this.tryFlush({ trigger: 'add' });
}
} catch (error) {
this.logger.error({ err: error, input }, 'Failed to add group');
@@ -163,13 +156,17 @@ export class GroupBuffer extends BaseBuffer {
}
async processBuffer(): Promise<void> {
+ await this.drainBatches(this.batchSize, () => this.processBatch());
+ }
+
+ private async processBatch(): Promise<number> {
const lrangeStart = performance.now();
const items = await this.redis.lrange(this.redisKey, 0, this.batchSize - 1);
const lrangeMs = performance.now() - lrangeStart;
if (items.length === 0) {
this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
- return;
+ return 0;
}
// Raw passthrough: each Redis entry is already a JSONEachRow line.
@@ -193,5 +190,6 @@ export class GroupBuffer extends BaseBuffer {
rowsProcessed: items.length,
phases: { lrangeMs, chInsertMs, trimMs },
});
+ return items.length;
}
}
diff --git a/packages/core/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.ts
index b4910488d..6784083e3 100644
--- a/packages/core/src/buffers/profile-buffer.test.ts
+++ b/packages/core/src/buffers/profile-buffer.test.ts
@@ -1,6 +1,7 @@
import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
import { getRedisCache } from '@openpanel/redis';
import { bufferDepsWithCh } from '../../test/buffer-fixtures';
+import { testCoreConfig } from '../../test/config-fixture';
import type { IClickhouseProfile } from '../modules/profile/profile.service';
// The client comes in as `BufferDeps.ch` and reads go through core's own
@@ -80,6 +81,24 @@ describe('ProfileBuffer', () => {
expect(sizeAfter).toBe(sizeBefore + 1);
});
+ it('add never flushes; a cron flush drains past one batch', async () => {
+ const config = testCoreConfig();
+ config.buffers.profile.batchSize = 2;
+ const buffer = new ProfileBuffer(
+ bufferDepsWithCh({ insert: chInsert }, config)
+ );
+
+ for (let i = 0; i < 5; i++) {
+ await buffer.add(makeProfile({ id: `profile-${i}` }));
+ }
+ expect(chInsert).not.toHaveBeenCalled();
+
+ await buffer.tryFlush({ trigger: 'cron' });
+
+ expect(chInsert).toHaveBeenCalledTimes(3);
+ expect(await buffer.getBufferSize()).toBe(0);
+ });
+
it('concurrent adds: both raw profiles are queued', async () => {
const identifyProfile = makeProfile({
first_name: 'John',
diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.ts
index d4f447d8b..3660cf2a3 100644
--- a/packages/core/src/buffers/profile-buffer.ts
+++ b/packages/core/src/buffers/profile-buffer.ts
@@ -119,21 +119,7 @@ export class ProfileBuffer extends BaseBuffer {
}
}
- const result = await this.redis
- .multi()
- .rpush(this.redisKey, JSON.stringify(profile))
- .llen(this.redisKey)
- .exec();
-
- if (!result) {
- this.logger.error({ profile }, 'Failed to add profile to Redis');
- return;
- }
-
- const bufferLength = (result?.[1]?.[1] as number) ?? 0;
- if (bufferLength >= this.batchSize) {
- await this.tryFlush({ trigger: 'add' });
- }
+ await this.redis.rpush(this.redisKey, JSON.stringify(profile));
} catch (error) {
this.logger.error({ err: error, profile }, 'Failed to add profile');
}
@@ -241,6 +227,10 @@ export class ProfileBuffer extends BaseBuffer {
}
async processBuffer() {
+ await this.drainBatches(this.batchSize, () => this.processBatch());
+ }
+
+ private async processBatch(): Promise<number> {
const lrangeStart = performance.now();
const rawProfiles = await this.redis.lrange(
this.redisKey,
@@ -251,7 +241,7 @@ export class ProfileBuffer extends BaseBuffer {
if (rawProfiles.length === 0) {
this.reportFlushStats({ rowsProcessed: 0, phases: { lrangeMs } });
- return;
+ return 0;
}
// Yields to the event loop periodically so concurrent add() calls and
@@ -367,5 +357,6 @@ export class ProfileBuffer extends BaseBuffer {
rowsProcessed: rawProfiles.length,
phases: { lrangeMs, chFetchMs, chInsertMs, trimMs },
});
+ return rawProfiles.length;
}
}
diff --git a/packages/core/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.ts
index c3b94f74e..dc097c9f7 100644
--- a/packages/core/src/buffers/session-buffer.ts
+++ b/packages/core/src/buffers/session-buffer.ts
@@ -81,8 +81,6 @@ function pickUtm(
const DEFAULT_BATCH_SIZE = 1000;
const DEFAULT_CHUNK_SIZE = 1000;
-/** One cron run drains as many batches as fit before the next run is due. */
-const DRAIN_TIME_BUDGET_MS = 8000;
export class SessionBuffer extends BaseBuffer {
private readonly batchSize =
@@ -501,18 +499,7 @@ export class SessionBuffer extends BaseBuffer {
}
async processBuffer() {
- const deadline = performance.now() + DRAIN_TIME_BUDGET_MS;
- let rowsProcessed = 0;
-
- while (performance.now() < deadline) {
- const processed = await this.processBatch();
- rowsProcessed += processed;
- if (processed < this.batchSize) {
- break;
- }
- }
-
- this.reportFlushStats({ rowsProcessed });
+ await this.drainBatches(this.batchSize, () => this.processBatch());
}
/** Inserts and trims one batch from the head of the list; returns its size. */
@@ -561,6 +548,7 @@ export class SessionBuffer extends BaseBuffer {
// (and the next flush retries the still-queued rows).
await this.redis.ltrim(this.redisKey, events.length, -1);
+ this.reportFlushStats({ rowsProcessed: events.length });
this.logger.debug({ count: events.length }, 'Processed sessions');
return events.length;
}
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.ts
index c0a4d3bbc..a1ddc3bb7 100644
--- a/packages/core/src/jobs.registry.test.ts
+++ b/packages/core/src/jobs.registry.test.ts
@@ -7,6 +7,7 @@ const GOLDEN_SCHEDULER_IDS = [
'cohortRefresh',
'dataHealth',
'delete',
+ 'flushBots',
'flushEvents',
'flushExports',
'flushGroups',
@@ -108,6 +109,7 @@ test('the cron queue carries the insight module cron fragment', () => {
'cohortRefresh',
'dataHealth',
'delete',
+ 'flushBots',
'flushExports',
'flushEvents',
'flushGroups',
@@ -221,7 +223,7 @@ test('the notification queue carries the notification module job', () => {
});
});
-test('the cron queue derives the golden 20 scheduler ids', () => {
+test('the cron queue derives the golden 21 scheduler ids', () => {
const derivedIds = [
...schedulersFromRegistry(queues.cron).map((s) => s.id),
PING_SCHEDULE.id,
diff --git a/packages/core/src/jobs/schedulers.test.ts b/packages/core/src/jobs/schedulers.test.ts
index 3a70c983b..c66fc724b 100644
--- a/packages/core/src/jobs/schedulers.test.ts
+++ b/packages/core/src/jobs/schedulers.test.ts
@@ -7,12 +7,13 @@ import {
startSchedulers,
} from './schedulers';
-// Byte-identity snapshot: these 20 scheduler ids must not change without a
+// Byte-identity snapshot: these 21 scheduler ids must not change without a
// migration, since BullMQ persists them as job-scheduler keys in Redis.
const GOLDEN_SCHEDULER_IDS = [
'cohortRefresh',
'dataHealth',
'delete',
+ 'flushBots',
'flushEvents',
'flushExports',
'flushGroups',
@@ -99,7 +100,7 @@ function fakeQueue(options?: {
test('the golden scheduler ids are reproduced byte-for-byte', () => {
const allIds = [...CRON_SCHEDULES.map((s) => s.id), PING_SCHEDULE.id].sort();
- expect(allIds).toHaveLength(20);
+ expect(allIds).toHaveLength(21);
expect(allIds).toEqual(GOLDEN_SCHEDULER_IDS);
});
@@ -121,7 +122,7 @@ test('ping is excluded by default — neither self-hosted nor production', async
});
const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();
- expect(upsertedIds).toHaveLength(19);
+ expect(upsertedIds).toHaveLength(20);
expect(upsertedIds).not.toContain('ping');
});
@@ -162,7 +163,7 @@ test('ping is included only when self-hosted AND production', async () => {
});
const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();
- expect(upsertedIds).toHaveLength(20);
+ expect(upsertedIds).toHaveLength(21);
expect(upsertedIds).toEqual(GOLDEN_SCHEDULER_IDS);
});
diff --git a/packages/core/src/modules/event/event.jobs.ts b/packages/core/src/modules/event/event.jobs.ts
index 397cfcde0..3882e3e70 100644
--- a/packages/core/src/modules/event/event.jobs.ts
+++ b/packages/core/src/modules/event/event.jobs.ts
@@ -1,9 +1,11 @@
-// The event buffer (Redis -> ClickHouse `events`) is this module's domain.
+// The event buffer (Redis -> ClickHouse `events`) and the bot buffer
+// (-> `events_bots`) are this module's domain.
import { z } from 'zod';
import { defineJob } from '../../jobs/define';
const FLUSH_EVENTS_INTERVAL_MS = 10_000;
+const FLUSH_BOTS_INTERVAL_MS = 30_000;
/** This module's fragment of the `cron` queue's jobs. */
export const eventCronJobs = {
@@ -14,4 +16,11 @@ export const eventCronJobs = {
await ctx.buffers.event.tryFlush({ trigger: 'cron' });
},
}),
+ flushBots: defineJob({
+ payload: z.null(),
+ cron: { every: FLUSH_BOTS_INTERVAL_MS },
+ handler: async ({ ctx }) => {
+ await ctx.buffers.bot.tryFlush({ trigger: 'cron' });
+ },
+ }),
};