OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): add bot events with buffer
Carl-Gerhard Lindesvärd46fe17e558fa
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex cc39ee9c8..3d77e99bb 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -54,6 +54,7 @@ const startServer = async () => {const fastify = Fastify({maxParamLength: 15_000,bodyLimit: 1048576 * 500, // 500MB+ logger,});fastify.register(compress, {diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex 6d0af53a8..c667e99a3 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -67,7 +67,7 @@ queues.forEach((queue) => {});// Buffer-const buffers = ['events_v2', 'profiles'];+const buffers = ['events_v2', 'profiles', 'events_bots'];buffers.forEach((buffer) => {register.registerMetric(diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.tsnew file mode 100644index 000000000..48b1f678e--- /dev/null+++ b/packages/db/src/buffers/bot-buffer.ts@@ -0,0 +1,39 @@+import { ch, TABLE_NAMES } from '../clickhouse-client';+import type { IClickhouseBotEvent } from '../services/event.service';+import type {+ Find,+ FindMany,+ OnCompleted,+ OnInsert,+ ProcessQueue,+} from './buffer';+import { RedisBuffer } from './buffer';++export class BotBuffer extends RedisBuffer<IClickhouseBotEvent> {+ constructor() {+ super({+ table: TABLE_NAMES.events_bots,+ batchSize: 100,+ });+ }++ public onInsert?: OnInsert<IClickhouseBotEvent> | undefined;+ public onCompleted?: OnCompleted<IClickhouseBotEvent> | undefined;++ public processQueue: ProcessQueue<IClickhouseBotEvent> = async (queue) => {+ await ch.insert({+ table: TABLE_NAMES.events_bots,+ values: queue.map((item) => item.event),+ format: 'JSONEachRow',+ });+ return queue.map((item) => item.index);+ };++ public findMany: FindMany<IClickhouseBotEvent, IClickhouseBotEvent> = () => {+ return Promise.resolve([]);+ };++ public find: Find<IClickhouseBotEvent, IClickhouseBotEvent> = () => {+ return Promise.resolve(null);+ };+}diff --git a/packages/db/src/buffers/buffer.ts b/packages/db/src/buffers/buffer.tsindex 1ec48a104..9554f2d7b 100644--- a/packages/db/src/buffers/buffer.ts+++ b/packages/db/src/buffers/buffer.ts@@ -1,5 +1,12 @@import { getRedisCache } from '@openpanel/redis';+const logger = {+ debug: (...args: unknown[]) => console.log('[DEBUG]', ...args),+ info: (...args: unknown[]) => console.log('[INFO]', ...args),+ warn: (...args: unknown[]) => console.log('[WARN]', ...args),+ error: (...args: unknown[]) => console.log('[ERROR]', ...args),+};+export const DELETE = '__DELETE__';export type QueueItem<T> = {@@ -66,7 +73,14 @@ export abstract class RedisBuffer<T> {await getRedisCache().rpush(this.getKey(), JSON.stringify(value));const length = await getRedisCache().llen(this.getKey());+ logger.debug(+ `Inserted item into buffer ${this.table}. Current length: ${length}`+ );+if (this.batchSize && length >= this.batchSize) {+ logger.info(+ `Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...`+ );this.flush();}}@@ -76,12 +90,12 @@ export abstract class RedisBuffer<T> {const queue = await this.getQueue(this.batchSize || -1);if (queue.length === 0) {- return {- count: 0,- data: [],- };+ logger.debug(`Flush called on empty buffer ${this.table}`);+ return { count: 0, data: [] };}+ logger.info(`Flushing ${queue.length} items from buffer ${this.table}`);+try {const indexes = await this.processQueue(queue);await this.deleteIndexes(indexes);@@ -91,19 +105,19 @@ export abstract class RedisBuffer<T> {if (this.onCompleted) {const res = await this.onCompleted(data);- return {- count: res.length,- data: res,- };+ logger.info(+ `Completed processing ${res.length} items from buffer ${this.table}`+ );+ return { count: res.length, data: res };}- return {- count: indexes.length,- data: indexes,- };+ logger.info(+ `Processed ${indexes.length} items from buffer ${this.table}`+ );+ return { count: indexes.length, data: indexes };} catch (e) {- console.log(- `[${this.getKey()}] Failed to processQueue while flushing:`,+ logger.error(+ `Failed to process queue while flushing buffer ${this.table}:`,e);const timestamp = new Date().getTime();@@ -112,9 +126,15 @@ export abstract class RedisBuffer<T> {data: JSON.stringify(queue.map((item) => item.event)),retries: 0,});+ logger.warn(+ `Stored ${queue.length} failed items in ${this.getKey(`failed:${timestamp}`)}`+ );}} catch (e) {- console.log(`[${this.getKey()}] Failed to getQueue while flushing:`, e);+ logger.error(+ `Failed to get queue while flushing buffer ${this.table}:`,+ e+ );}}@@ -125,22 +145,26 @@ export abstract class RedisBuffer<T> {});multi.lrem(this.getKey(), 0, DELETE);await multi.exec();+ logger.debug(`Deleted ${indexes.length} items from buffer ${this.table}`);}public async getQueue(limit: number): Promise<QueueItem<T>[]> {const queue = await getRedisCache().lrange(this.getKey(), 0, limit);- return queue+ const result = queue.map((item, index) => ({event: this.transformQueueItem(item),index,})).filter((item): item is QueueItem<T> => item.event !== null);+ logger.debug(`Retrieved ${result.length} items from buffer ${this.table}`);+ return result;}private transformQueueItem(item: string): T | null {try {return JSON.parse(item);} catch (e) {+ logger.warn(`Failed to parse item in buffer ${this.table}:`, e);return null;}}diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex 7623e9415..69909f097 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -1,5 +1,7 @@+import { BotBuffer } from './bot-buffer';import { EventBuffer } from './event-buffer';import { ProfileBuffer } from './profile-buffer';export const eventBuffer = new EventBuffer();export const profileBuffer = new ProfileBuffer();+export const botBuffer = new BotBuffer();diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 763235fd9..8c05f0658 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -3,7 +3,7 @@ import { mergeDeepRight } from 'ramda';import { toDots } from '@openpanel/common';import { getRedisCache } from '@openpanel/redis';-import { ch, chQuery } from '../clickhouse-client';+import { ch, chQuery, TABLE_NAMES } from '../clickhouse-client';import type {IClickhouseProfile,IServiceProfile,@@ -22,7 +22,7 @@ import { RedisBuffer } from './buffer';export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {constructor() {super({- table: 'profiles',+ table: TABLE_NAMES.profiles,batchSize: 100,});}@@ -56,7 +56,7 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {);await ch.insert({- table: 'profiles',+ table: TABLE_NAMES.profiles,values: cleanedQueue.map((item) => {const profile = profiles.find((p) =>diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 1cdb276b0..428a05ca8 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -6,7 +6,7 @@ import { toDots } from '@openpanel/common';import { cacheable, getRedisCache } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';-import { eventBuffer } from '../buffers';+import { botBuffer, eventBuffer } from '../buffers';import {ch,chQuery,@@ -37,6 +37,26 @@ export type IServicePage = {origin: string;};+export interface IClickhouseBotEvent {+ id: string;+ name: string;+ type: string;+ project_id: string;+ path: string;+ created_at: string;+}++export interface IServiceBotEvent {+ id: string;+ name: string;+ type: string;+ projectId: string;+ path: string;+ createdAt: Date;+}++export type IServiceCreateBotEventPayload = Omit<IServiceBotEvent, 'id'>;+export interface IClickhouseEvent {id: string;name: string;@@ -535,33 +555,20 @@ export async function getEventsCount({return res[0]?.count ?? 0;}-interface CreateBotEventPayload {- name: string;- type: string;- path: string;- projectId: string;- createdAt: Date;-}-export function createBotEvent({name,type,projectId,createdAt,path,-}: CreateBotEventPayload) {- return ch.insert({- table: 'events_bots',- format: 'JSONEachRow',- values: [- {- name,- type,- project_id: projectId,- path,- created_at: formatClickhouseDate(createdAt),- },- ],+}: IServiceCreateBotEventPayload) {+ return botBuffer.insert({+ id: uuid(),+ name,+ type,+ project_id: projectId,+ path,+ created_at: formatClickhouseDate(createdAt),});}