OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feature(worker+api): improve buffer
Carl-Gerhard Lindesvärd545cbfd520d8
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex 167941fea..53893315b 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -6,6 +6,7 @@ import { Worker } from 'bullmq';import express from 'express';import { createInitialSalts } from '@openpanel/db';+import type { CronQueueType } from '@openpanel/queue';import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';import { getRedisQueue } from '@openpanel/redis';@@ -137,70 +138,75 @@ async function start() {}),);- await cronQueue.add(- 'salt',+ const jobs: {+ name: string;+ type: CronQueueType;+ pattern: string | number;+ }[] = [{+ name: 'salt',type: 'salt',- payload: undefined,+ pattern: '0 0 * * *',},{- jobId: 'salt',- repeat: {- utc: true,- pattern: '0 0 * * *',- },- },- );-- await cronQueue.add(- 'flush',- {+ name: 'flush',type: 'flushEvents',- payload: undefined,- },- {- jobId: 'flushEvents',- repeat: {- every: process.env.BATCH_INTERVAL- ? Number.parseInt(process.env.BATCH_INTERVAL, 10)- : 1000 * 10,- },+ pattern: 1000 * 10,},- );-- await cronQueue.add(- 'flush',{+ name: 'flush',type: 'flushProfiles',- payload: undefined,+ pattern: 1000 * 60 * 30,},- {- jobId: 'flushProfiles',- repeat: {- every: 2 * 1000,- },- },- );+ ];if (process.env.SELF_HOSTED && process.env.NODE_ENV === 'production') {+ jobs.push({+ name: 'ping',+ type: 'ping',+ pattern: '0 0 * * *',+ });+ }++ // Add repeatable jobs+ for (const job of jobs) {await cronQueue.add(- 'ping',+ job.name,{- type: 'ping',+ type: job.type,payload: undefined,},{- jobId: 'ping',- repeat: {- pattern: '0 0 * * *',- },+ jobId: job.type,+ repeat:+ typeof job.pattern === 'number'+ ? {+ every: job.pattern,+ }+ : {+ pattern: job.pattern,+ },},);}+ // Remove outdated repeatable jobsconst repeatableJobs = await cronQueue.getRepeatableJobs();-- logger.info('Repeatable jobs', { repeatableJobs });+ for (const repeatableJob of repeatableJobs) {+ const match = jobs.find(+ (job) => `${job.name}:${job.type}:::${job.pattern}` === repeatableJob.key,+ );+ if (match) {+ logger.info('Repeatable job exists', {+ key: repeatableJob.key,+ });+ } else {+ logger.info('Removing repeatable job', {+ key: repeatableJob.key,+ });+ cronQueue.removeRepeatableByKey(repeatableJob.key);+ }+ }await createInitialSalts();}diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.tsindex a12e0f3d4..5c0057ccb 100644--- a/apps/worker/src/jobs/cron.ts+++ b/apps/worker/src/jobs/cron.ts@@ -12,10 +12,10 @@ export async function cronJob(job: Job<CronQueuePayload>) {return await salt();}case 'flushEvents': {- return await eventBuffer.flush();+ return await eventBuffer.tryFlush();}case 'flushProfiles': {- return await profileBuffer.flush();+ return await profileBuffer.tryFlush();}case 'ping': {return await ping();diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 6dc2b372c..a095a3875 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -15,7 +15,7 @@ export async function createSessionEnd() {const payload = job.data.payload;const eventsInBuffer = await eventBuffer.findMany(- (item) => item.event.session_id === payload.sessionId,+ (item) => item.session_id === payload.sessionId,);const sql = `diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex ab0303824..93a8093b4 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -9,12 +9,14 @@ import type { IServiceCreateEventPayload } from '@openpanel/db';import { createEvent } from '@openpanel/db';import { getLastScreenViewFromProfileId } from '@openpanel/db/src/services/event.service';import { findJobByPrefix, sessionsQueue } from '@openpanel/queue';-import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';+import type {+ EventsQueuePayloadCreateSessionEnd,+ EventsQueuePayloadIncomingEvent,+} from '@openpanel/queue';import { getRedisQueue } from '@openpanel/redis';const GLOBAL_PROPERTIES = ['__path', '__referrer'];-const SESSION_TIMEOUT = 1000 * 60 * 30;-const SESSION_END_TIMEOUT = SESSION_TIMEOUT + 1000;+export const SESSION_TIMEOUT = 1000 * 60 * 30;export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {const {@@ -100,37 +102,14 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {previousDeviceId,});- const sessionEndPayload = sessionEnd?.job.data.payload || {- sessionId: uuid(),- deviceId: currentDeviceId,- profileId,- projectId,- };-- const sessionEndJobId =- sessionEnd?.job.id ??- `sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${Date.now()}`;-- if (sessionEnd) {- // If for some reason we have a session end job that is not a createSessionEnd job- if (sessionEnd.job.data.type !== 'createSessionEnd') {- throw new Error('Invalid session end job');- }-- await sessionEnd.job.changeDelay(SESSION_TIMEOUT);- } else {- await sessionsQueue.add(- 'session',- {- type: 'createSessionEnd',- payload: sessionEndPayload,- },- {- delay: SESSION_END_TIMEOUT,- jobId: sessionEndJobId,- },- );- }+ const sessionEndPayload =+ sessionEnd?.job.data.payload ||+ ({+ sessionId: uuid(),+ deviceId: currentDeviceId,+ profileId,+ projectId,+ } satisfies EventsQueuePayloadCreateSessionEnd['payload']);const payload: IServiceCreateEventPayload = {name: body.name,@@ -158,13 +137,46 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {duration: 0,path: path,origin: origin,- referrer: referrer?.url,- referrerName: referrer?.name || utmReferrer?.name || '',- referrerType: referrer?.type || utmReferrer?.type || '',+ referrer: sessionEndPayload.referrer || referrer?.url,+ referrerName:+ sessionEndPayload.referrerName ||+ referrer?.name ||+ utmReferrer?.name ||+ '',+ referrerType:+ sessionEndPayload.referrerType ||+ referrer?.type ||+ utmReferrer?.type ||+ '',sdkName,sdkVersion,};+ const sessionEndJobId =+ sessionEnd?.job.id ??+ `sessionEnd:${projectId}:${sessionEndPayload.deviceId}:${getTime(createdAt)}`;++ if (sessionEnd) {+ // If for some reason we have a session end job that is not a createSessionEnd job+ if (sessionEnd.job.data.type !== 'createSessionEnd') {+ throw new Error('Invalid session end job');+ }++ await sessionEnd.job.changeDelay(SESSION_TIMEOUT);+ } else {+ await sessionsQueue.add(+ 'session',+ {+ type: 'createSessionEnd',+ payload,+ },+ {+ delay: SESSION_TIMEOUT,+ jobId: sessionEndJobId,+ },+ );+ }+if (!sessionEnd) {await createEvent({...payload,diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsnew file mode 100644index 000000000..aab2733ce--- /dev/null+++ b/apps/worker/src/jobs/events.incoming-events.test.ts@@ -0,0 +1,351 @@+import { type Mock, beforeEach, describe, expect, it, mock } from 'bun:test';+import { getTime, toISOString } from '@openpanel/common';+import type { Job } from 'bullmq';+import { SESSION_TIMEOUT, incomingEvent } from './events.incoming-event';++const projectId = 'test-project';+const currentDeviceId = 'device-123';+const previousDeviceId = 'device-456';+const geo = {+ country: 'US',+ city: 'New York',+ region: 'NY',+ longitude: 0,+ latitude: 0,+};++const createEvent = mock(() => {});+const getLastScreenViewFromProfileId = mock();+// // Mock dependencies+mock.module('@openpanel/db', () => ({+ createEvent,+ getLastScreenViewFromProfileId,+}));++const sessionsQueue = { add: mock(() => Promise.resolve({})) };++const findJobByPrefix = mock();++mock.module('@openpanel/queue', () => ({+ sessionsQueue,+ findJobByPrefix,+}));++const getRedisQueue = mock(() => ({+ keys: mock(() => Promise.resolve([])),+}));++mock.module('@openpanel/redis', () => ({+ getRedisQueue,+}));++describe('incomingEvent', () => {+ beforeEach(() => {+ createEvent.mockClear();+ findJobByPrefix.mockClear();+ sessionsQueue.add.mockClear();+ getLastScreenViewFromProfileId.mockClear();+ });++ it('should create a session start and an event', async () => {+ const timestamp = new Date();+ // Mock job data+ const jobData = {+ payload: {+ geo,+ event: {+ name: 'test_event',+ timestamp: timestamp.toISOString(),+ properties: { __path: 'https://example.com/test' },+ },+ headers: {+ 'user-agent':+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',+ 'openpanel-sdk-name': 'web',+ 'openpanel-sdk-version': '1.0.0',+ },+ projectId,+ currentDeviceId,+ previousDeviceId,+ priority: true,+ },+ };++ const job = { data: jobData } as Job;++ // Execute the job+ await incomingEvent(job);++ const event = {+ name: 'test_event',+ deviceId: currentDeviceId,+ // @ts-expect-error+ sessionId: createEvent.mock.calls[1][0].sessionId,+ profileId: '',+ projectId,+ properties: {+ __hash: undefined,+ __query: undefined,+ },+ createdAt: timestamp,+ country: 'US',+ city: 'New York',+ region: 'NY',+ longitude: 0,+ latitude: 0,+ os: 'Windows',+ osVersion: '10',+ browser: 'Chrome',+ browserVersion: '91.0.4472.124',+ device: 'desktop',+ brand: '',+ model: '',+ duration: 0,+ path: '/test',+ origin: 'https://example.com',+ referrer: '',+ referrerName: '',+ referrerType: 'unknown',+ sdkName: 'web',+ sdkVersion: '1.0.0',+ };++ expect(sessionsQueue.add.mock.calls[0]).toMatchObject([+ 'session',+ {+ type: 'createSessionEnd',+ payload: event,+ },+ {+ delay: SESSION_TIMEOUT,+ jobId: `sessionEnd:${projectId}:${event.deviceId}:${timestamp.getTime()}`,+ },+ ]);++ // Assertions+ // Issue: https://github.com/oven-sh/bun/issues/10380+ // expect(createEvent).toHaveBeenCalledWith(...)+ expect(createEvent.mock.calls[0]).toMatchObject([+ {+ name: 'session_start',+ deviceId: currentDeviceId,+ sessionId: expect.stringMatching(+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i,+ ),+ profileId: '',+ projectId,+ properties: {+ __hash: undefined,+ __query: undefined,+ },+ createdAt: new Date(timestamp.getTime() - 100),+ country: 'US',+ city: 'New York',+ region: 'NY',+ longitude: 0,+ latitude: 0,+ os: 'Windows',+ osVersion: '10',+ browser: 'Chrome',+ browserVersion: '91.0.4472.124',+ device: 'desktop',+ brand: '',+ model: '',+ duration: 0,+ path: '/test',+ origin: 'https://example.com',+ referrer: '',+ referrerName: '',+ referrerType: 'unknown',+ sdkName: 'web',+ sdkVersion: '1.0.0',+ },+ ]);+ expect(createEvent.mock.calls[1]).toMatchObject([event]);++ // Add more specific assertions based on the expected behavior+ });++ it('should reuse existing session', async () => {+ // Mock job data+ const jobData = {+ payload: {+ geo,+ event: {+ name: 'test_event',+ timestamp: new Date().toISOString(),+ properties: { __path: 'https://example.com/test' },+ },+ headers: {+ 'user-agent':+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',+ 'openpanel-sdk-name': 'web',+ 'openpanel-sdk-version': '1.0.0',+ },+ projectId,+ currentDeviceId,+ previousDeviceId,+ priority: false,+ },+ };+ const changeDelay = mock();+ findJobByPrefix.mockReturnValueOnce({+ changeDelay,+ data: {+ type: 'createSessionEnd',+ payload: {+ sessionId: 'session-123',+ deviceId: currentDeviceId,+ profileId: currentDeviceId,+ projectId,+ },+ },+ });++ const job = { data: jobData } as Job;++ // Execute the job+ await incomingEvent(job);++ expect(changeDelay.mock.calls[0]).toMatchObject([SESSION_TIMEOUT]);++ // Assertions+ // Issue: https://github.com/oven-sh/bun/issues/10380+ // expect(createEvent).toHaveBeenCalledWith(...)+ expect(createEvent.mock.calls[0]).toMatchObject([+ {+ name: 'test_event',+ deviceId: currentDeviceId,+ profileId: '',+ sessionId: 'session-123',+ projectId,+ properties: {+ __hash: undefined,+ __query: undefined,+ },+ createdAt: expect.any(Date),+ country: 'US',+ city: 'New York',+ region: 'NY',+ longitude: 0,+ latitude: 0,+ os: 'Windows',+ osVersion: '10',+ browser: 'Chrome',+ browserVersion: '91.0.4472.124',+ device: 'desktop',+ brand: '',+ model: '',+ duration: 0,+ path: '/test',+ origin: 'https://example.com',+ referrer: '',+ referrerName: '',+ referrerType: 'unknown',+ sdkName: 'web',+ sdkVersion: '1.0.0',+ },+ ]);++ // Add more specific assertions based on the expected behavior+ });++ it('should handle server events', async () => {+ const timestamp = new Date();+ const jobData = {+ payload: {+ geo,+ event: {+ name: 'server_event',+ timestamp: timestamp.toISOString(),+ properties: { custom_property: 'test_value' },+ profileId: 'profile-123',+ },+ headers: {+ 'user-agent': 'OpenPanel Server/1.0',+ 'openpanel-sdk-name': 'server',+ 'openpanel-sdk-version': '1.0.0',+ },+ projectId,+ currentDeviceId: '',+ previousDeviceId: '',+ priority: true,+ },+ };++ const job = { data: jobData } as Job;++ const mockLastScreenView = {+ deviceId: 'last-device-123',+ sessionId: 'last-session-456',+ country: 'CA',+ city: 'Toronto',+ region: 'ON',+ os: 'iOS',+ osVersion: '15.0',+ browser: 'Safari',+ browserVersion: '15.0',+ device: 'mobile',+ brand: 'Apple',+ model: 'iPhone',+ path: '/last-path',+ origin: 'https://example.com',+ referrer: 'https://google.com',+ referrerName: 'Google',+ referrerType: 'search',+ };++ getLastScreenViewFromProfileId.mockReturnValueOnce(mockLastScreenView);++ await incomingEvent(job);++ // expect(getLastScreenViewFromProfileId).toHaveBeenCalledWith({+ // profileId: 'profile-123',+ // projectId,+ // });++ expect(createEvent.mock.calls[0]).toMatchObject([+ {+ name: 'server_event',+ deviceId: 'last-device-123',+ sessionId: 'last-session-456',+ profileId: 'profile-123',+ projectId,+ properties: {+ custom_property: 'test_value',+ user_agent: 'OpenPanel Server/1.0',+ },+ createdAt: timestamp,+ country: 'CA',+ city: 'Toronto',+ region: 'ON',+ longitude: 0,+ latitude: 0,+ os: 'iOS',+ osVersion: '15.0',+ browser: 'Safari',+ browserVersion: '15.0',+ device: 'mobile',+ brand: 'Apple',+ model: 'iPhone',+ duration: 0,+ path: '/last-path',+ origin: 'https://example.com',+ referrer: 'https://google.com',+ referrerName: 'Google',+ referrerType: 'search',+ sdkName: 'server',+ sdkVersion: '1.0.0',+ },+ ]);++ expect(sessionsQueue.add).not.toHaveBeenCalled();+ expect(findJobByPrefix).not.toHaveBeenCalled();+ });++ // Add more test cases for different scenarios:+ // - Server events+ // - Existing sessions+ // - Different priorities+ // - Error cases+});diff --git a/apps/worker/src/utils/parse-user-agent.ts b/apps/worker/src/utils/parse-user-agent.tsindex b7e8666d3..42ddc8bb2 100644--- a/apps/worker/src/utils/parse-user-agent.ts+++ b/apps/worker/src/utils/parse-user-agent.ts@@ -9,7 +9,9 @@ export function parseUserAgent(ua?: string | null) {if (!ua) return parsedServerUa;const res = new UAParser(ua).getResult();- if (isServer(ua)) return parsedServerUa;+ if (isServer(ua)) {+ return parsedServerUa;+ }return {os: res.os.name,@@ -77,7 +79,9 @@ function isServer(userAgent: string) {return true;}- return !!userAgent.match(/^([^\s]+\/[\d.]+\s*)+$/);+ // Matches user agents like "Go-http-client/1.0" or "Go Http Client/1.0"+ // It should just match the first name (with optional spaces) and version+ return !!userAgent.match(/^[^\/]+\/[\d.]+$/);}export function getDevice(ua: string) {diff --git a/packages/common/src/url.ts b/packages/common/src/url.tsindex 43d31ef0a..1ae1358dc 100644--- a/packages/common/src/url.ts+++ b/packages/common/src/url.ts@@ -22,12 +22,13 @@ export function parsePath(path?: string): {}try {- const url = new URL(path);+ const hasOrigin = path.startsWith('http');+ const url = new URL(path, hasOrigin ? undefined : 'http://localhost');return {query: parseSearchParams(url.searchParams),path: url.pathname,hash: url.hash || undefined,- origin: url.origin,+ origin: hasOrigin ? url.origin : '',};} catch (error) {return {diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.tsindex 576464684..0d820b36a 100644--- a/packages/db/src/buffers/bot-buffer.ts+++ b/packages/db/src/buffers/bot-buffer.ts@@ -1,39 +1,18 @@import { TABLE_NAMES, ch } from '../clickhouse-client';import type { IClickhouseBotEvent } from '../services/event.service';-import type {- Find,- FindMany,- OnCompleted,- OnInsert,- ProcessQueue,-} from './buffer';import { RedisBuffer } from './buffer';-export class BotBuffer extends RedisBuffer<IClickhouseBotEvent> {+type BufferType = IClickhouseBotEvent;+export class BotBuffer extends RedisBuffer<BufferType> {constructor() {- super({- table: TABLE_NAMES.events_bots,- batchSize: 100,- });+ super(TABLE_NAMES.events, 500);}- public onInsert?: OnInsert<IClickhouseBotEvent> | undefined;- public onCompleted?: OnCompleted<IClickhouseBotEvent> | undefined;-- public processQueue: ProcessQueue<IClickhouseBotEvent> = async (queue) => {+ protected async insertIntoDB(items: BufferType[]): Promise<void> {await ch.insert({table: TABLE_NAMES.events_bots,- values: queue.map((item) => item.event),+ values: items,format: 'JSONEachRow',});- return queue.map((item) => item.index);- };-- public findMany: FindMany<IClickhouseBotEvent, IClickhouseBotEvent> = () => {- return Promise.resolve([]);- };-- public find: Find<IClickhouseBotEvent, IClickhouseBotEvent> = () => {- return Promise.resolve(null);- };+ }}diff --git a/packages/db/src/buffers/buffer.ts b/packages/db/src/buffers/buffer.tsindex 55dbc8c0e..908c3284c 100644--- a/packages/db/src/buffers/buffer.ts+++ b/packages/db/src/buffers/buffer.ts@@ -1,181 +1,219 @@+import { v4 as uuidv4 } from 'uuid';++import type { ILogger } from '@openpanel/logger';import { createLogger } from '@openpanel/logger';import { getRedisCache } from '@openpanel/redis';-export const DELETE = '__DELETE__';--export type QueueItem<T> = {- event: T;- index: number;-};--export type OnInsert<T> = (data: T) => unknown;--export type OnCompleted<T> =- | ((data: T[]) => Promise<unknown[]>)- | ((data: T[]) => unknown[]);--export type ProcessQueue<T> = (data: QueueItem<T>[]) => Promise<number[]>;-export type Find<T, R = unknown> = (- callback: (item: QueueItem<T>) => boolean,+ callback: (item: T) => boolean,) => Promise<R | null>;export type FindMany<T, R = unknown> = (- callback: (item: QueueItem<T>) => boolean,+ callback: (item: T) => boolean,) => Promise<R[]>;-const getError = (e: unknown) => {- if (e instanceof Error) {- return [- `Name: ${e.name}`,- `Message: ${e.message}`,- `Stack: ${e.stack}`,- `Cause: ${e.cause ? String(e.cause) : ''}`,- ].join('\n');- }- return 'Unknown error';-};--export abstract class RedisBuffer<T> {- // constructor- public prefix = 'op:buffer';- public table: string;- public batchSize?: number;- public logger: ReturnType<typeof createLogger>;- public disableAutoFlush?: boolean;-- // abstract methods- public abstract onInsert?: OnInsert<T>;- public abstract onCompleted?: OnCompleted<T>;- public abstract processQueue: ProcessQueue<T>;- public abstract find: Find<T, unknown>;- public abstract findMany: FindMany<T, unknown>;-- constructor(options: {- table: string;- batchSize?: number;- disableAutoFlush?: boolean;- }) {- this.table = options.table;- this.batchSize = options.batchSize;- this.disableAutoFlush = options.disableAutoFlush;+export class RedisBuffer<T> {+ protected prefix = 'op:buffer';+ protected bufferKey: string;+ private lockKey: string;+ protected maxBufferSize: number | null;+ protected logger: ILogger;++ constructor(bufferName: string, maxBufferSize: number | null) {+ this.bufferKey = bufferName;+ this.lockKey = `lock:${bufferName}`;+ this.maxBufferSize = maxBufferSize;this.logger = createLogger({ name: 'buffer' }).child({- table: this.table,+ buffer: bufferName,});}- public getKey(name?: string) {- const key = `${this.prefix}:${this.table}`;+ protected getKey(name?: string) {+ const key = `${this.prefix}:${this.bufferKey}`;if (name) {return `${key}:${name}`;}return key;}- public async insert(value: T) {- this.onInsert?.(value);- await getRedisCache().rpush(this.getKey(), JSON.stringify(value));+ async add(item: T): Promise<void> {+ try {+ this.onAdd(item);+ await getRedisCache().rpush(this.getKey(), JSON.stringify(item));+ const bufferSize = await getRedisCache().llen(this.getKey());++ this.logger.debug(`Item added. Current size: ${bufferSize}`);- const length = await getRedisCache().llen(this.getKey());- this.logger.debug(- `Inserted item into buffer ${this.table}. Current length: ${length}`,+ if (this.maxBufferSize && bufferSize >= this.maxBufferSize) {+ await this.tryFlush();+ }+ } catch (error) {+ this.logger.error('Failed to add item to buffer', { error, item });+ }+ }++ public async tryFlush(): Promise<void> {+ const lockId = uuidv4();+ const acquired = await getRedisCache().set(+ this.lockKey,+ lockId,+ 'EX',+ 8,+ 'NX',);- if (!this.disableAutoFlush && this.batchSize && length >= this.batchSize) {- this.logger.info(- `Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...`,- );- this.flush();+ if (acquired === 'OK') {+ this.logger.debug('Lock acquired. Attempting to flush.');+ try {+ await this.flush();+ } finally {+ await this.releaseLock(lockId);+ }+ } else {+ this.logger.debug('Failed to acquire lock for. Skipping flush.');}}- public async flush() {- try {- const queue = await this.getQueue(this.batchSize || -1);+ protected async waitForReleasedLock(+ maxWaitTime = 8000,+ checkInterval = 500,+ ): Promise<boolean> {+ const startTime = performance.now();- if (queue.length === 0) {- this.logger.debug(`Flush called on empty buffer ${this.table}`);- return { count: 0, data: [] };+ while (performance.now() - startTime < maxWaitTime) {+ const lock = await getRedisCache().get(this.lockKey);+ if (!lock) {+ return true;}- this.logger.info(- `Flushing ${queue.length} items from buffer ${this.table}`,- );+ await new Promise((resolve) => setTimeout(resolve, checkInterval));+ }+ this.logger.warn('Timeout waiting for lock release');+ return false;+ }++ private async retryOnce(cb: () => Promise<void>) {+ try {+ await cb();+ } catch (e) {+ this.logger.error(`#1 Failed to execute callback: ${cb.name}`, e);+ await new Promise((resolve) => setTimeout(resolve, 1000));try {- const indexes = await this.processQueue(queue);- await this.deleteIndexes(indexes);- const data = indexes- .map((index) => queue[index]?.event)- .filter((event): event is T => event !== null);-- if (this.onCompleted) {- const res = await this.onCompleted(data);- this.logger.info(- `Completed processing ${res.length} items from buffer ${this.table}`,- );- return { count: res.length, data: res };- }-- this.logger.info(- `Processed ${indexes.length} items from buffer ${this.table}`,- );- return { count: indexes.length, data: indexes };+ await cb();} catch (e) {- this.logger.error(- `Failed to process queue while flushing buffer ${this.table}:`,- e,- );- const timestamp = new Date().getTime();- await getRedisCache().hset(this.getKey(`failed:${timestamp}`), {- error: getError(e),- data: JSON.stringify(queue.map((item) => item.event)),- retries: 0,- });- this.logger.warn(- `Stored ${queue.length} failed items in ${this.getKey(`failed:${timestamp}`)}`,+ this.logger.error(`#2 Failed to execute callback: ${cb.name}`, e);+ }+ }+ }++ private async flush(): Promise<void> {+ // Use a transaction to ensure atomicity+ const result = await getRedisCache()+ .multi()+ .lrange(this.getKey(), 0, -1)+ .del(this.getKey())+ .exec();++ if (!result) {+ throw new Error('Redis transaction failed');+ }++ const lrange = result[0];++ if (!lrange || lrange[0] instanceof Error) {+ throw new Error('Redis transaction failed');+ }++ const items = lrange[1] as string[];++ const parsedItems = items.map((item) => JSON.parse(item) as T);++ if (parsedItems.length === 0) {+ this.logger.debug('No items to flush');+ return;+ }++ this.logger.info(`Flushing ${parsedItems.length} items`);++ try {+ const { toInsert, toKeep } = await this.processItems(parsedItems);++ if (toInsert.length) {+ await this.retryOnce(() => this.insertIntoDB(toInsert));+ this.onInsert(toInsert);+ }++ // Add back items to keep+ if (toKeep.length > 0) {+ await getRedisCache().lpush(+ this.getKey(),+ ...toKeep.map((item) => JSON.stringify(item)),);}- } catch (e) {- this.logger.error(- `Failed to get queue while flushing buffer ${this.table}:`,- e,++ this.logger.info(+ `Inserted ${toInsert.length} items into DB, kept ${toKeep.length} items in buffer`,+ {+ toInsert: toInsert.length,+ toKeep: toKeep.length,+ },);+ } catch (error) {+ this.logger.error('Failed to process queue while flushing buffer}:', {+ error,+ queueSize: items.length,+ });++ if (items.length > 0) {+ // Add back items to keep+ this.logger.debug('Adding all items back to buffer');+ await getRedisCache().lpush(+ this.getKey(),+ ...items.map((item) => JSON.stringify(item)),+ );+ }}}- public async deleteIndexes(indexes: number[]) {- const multi = getRedisCache().multi();- indexes.forEach((index) => {- multi.lset(this.getKey(), index, DELETE);- });- multi.lrem(this.getKey(), 0, DELETE);- await multi.exec();- this.logger.debug(- `Deleted ${indexes.length} items from buffer ${this.table}`,- );+ private async releaseLock(lockId: string): Promise<void> {+ this.logger.debug(`Released lock for ${this.getKey()}`);+ const script = `+ if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("del", KEYS[1])+ else+ return 0+ end+ `;+ await getRedisCache().eval(script, 1, this.lockKey, lockId);}- public async getQueue(limit: number): Promise<QueueItem<T>[]> {- const queue = await getRedisCache().lrange(this.getKey(), 0, limit);- const result = queue- .map((item, index) => ({- event: this.transformQueueItem(item),- index,- }))- .filter((item): item is QueueItem<T> => item.event !== null);- this.logger.debug(- `Retrieved ${result.length} items from buffer ${this.table}`,- );- return result;+ protected async getQueue(count?: number): Promise<T[]> {+ const items = await getRedisCache().lrange(this.getKey(), 0, count ?? -1);+ return items.map((item) => JSON.parse(item) as T);}- private transformQueueItem(item: string): T | null {- try {- return JSON.parse(item);- } catch (e) {- this.logger.warn(`Failed to parse item in buffer ${this.table}:`, e);- return null;- }+ protected processItems(items: T[]): Promise<{ toInsert: T[]; toKeep: T[] }> {+ return Promise.resolve({ toInsert: items, toKeep: [] });}++ protected insertIntoDB(_items: T[]): Promise<void> {+ throw new Error('Not implemented');+ }++ protected onAdd(_item: T): void {+ // Override in subclass+ }++ protected onInsert(_item: T[]): void {+ // Override in subclass+ }++ public findMany: FindMany<T, unknown> = () => {+ return Promise.resolve([]);+ };++ public find: Find<T, unknown> = () => {+ return Promise.resolve(null);+ };}diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 590050b6a..1253d5d8a 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -10,31 +10,16 @@ import type {IClickhouseEvent,IServiceEvent,} from '../services/event.service';-import type {- Find,- FindMany,- OnCompleted,- OnInsert,- ProcessQueue,- QueueItem,-} from './buffer';+import type { Find, FindMany } from './buffer';import { RedisBuffer } from './buffer';-const sortOldestFirst = (- a: QueueItem<IClickhouseEvent>,- b: QueueItem<IClickhouseEvent>,-) =>- new Date(a.event.created_at).getTime() -- new Date(b.event.created_at).getTime();--export class EventBuffer extends RedisBuffer<IClickhouseEvent> {+type BufferType = IClickhouseEvent;+export class EventBuffer extends RedisBuffer<BufferType> {constructor() {- super({- table: TABLE_NAMES.events,- });+ super(TABLE_NAMES.events, null);}- public onInsert?: OnInsert<IClickhouseEvent> | undefined = (event) => {+ public onAdd(event: BufferType) {getRedisPub().publish('event:received',SuperJSON.stringify(transformEvent(event)),@@ -47,24 +32,22 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {60 * 5,);}- };+ }- public onCompleted?: OnCompleted<IClickhouseEvent> | undefined = (- savedEvents,- ) => {- for (const event of savedEvents) {+ public onInsert(items: BufferType[]) {+ for (const event of items) {getRedisPub().publish('event:saved',SuperJSON.stringify(transformEvent(event)),);}+ }- return savedEvents.map((event) => event.id);- };-- public processQueue: ProcessQueue<IClickhouseEvent> = async (queue) => {- const itemsToClickhouse = new Set<QueueItem<IClickhouseEvent>>();- const itemsToStalled = new Set<QueueItem<IClickhouseEvent>>();+ protected async processItems(+ queue: BufferType[],+ ): Promise<{ toInsert: BufferType[]; toKeep: BufferType[] }> {+ const toInsert = new Set<BufferType>();+ const itemsToStalled = new Set<BufferType>();// Sort data by created_at// oldest first@@ -74,44 +57,37 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {// We only need screen_views since we want to calculate the duration of each screen// To do this we need a minimum of 2 screen_viewsqueue- .filter(- (item) =>- item.event.name !== 'screen_view' || item.event.device === 'server',- )- .forEach((item) => {+ .filter((item) => item.name !== 'screen_view' || item.device === 'server')+ .forEach((item, index) => {// Find the last event with data and merge it with the current event// We use profile_id here since this property can be set from backend as wellconst lastEventWithData = queue- .slice(0, item.index)+ .slice(0, index).findLast((lastEvent) => {return (- lastEvent.event.project_id === item.event.project_id &&- lastEvent.event.profile_id === item.event.profile_id &&- lastEvent.event.path !== ''+ lastEvent.project_id === item.project_id &&+ lastEvent.profile_id === item.profile_id &&+ lastEvent.path !== '');});- const event = deepMergeObjects<IClickhouseEvent>(- omit(['properties', 'duration'], lastEventWithData?.event || {}),- item.event,+ const event = deepMergeObjects<BufferType>(+ omit(['properties', 'duration'], lastEventWithData || {}),+ item,);if (lastEventWithData) {- // event.properties.__properties_from = lastEventWithData.event.id;+ event.properties.__properties_from = lastEventWithData.id;}- return itemsToClickhouse.add({- ...item,- event,- });+ return toInsert.add(event);});// Group screen_view events by session_idconst grouped = groupBy(- (item) => item.event.session_id,+ (item) => item.session_id,queue.filter(- (item) =>- item.event.name === 'screen_view' && item.event.device !== 'server',+ (item) => item.name === 'screen_view' && item.device !== 'server',),);@@ -123,9 +99,7 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {// If there is only one screen_view event we can send it back to redis since we can't calculate the durationconst hasSessionEnd = queue.find(- (item) =>- item.event.name === 'session_end' &&- item.event.session_id === sessionId,+ (item) => item.name === 'session_end' && item.session_id === sessionId,);screenViews@@ -136,20 +110,17 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {// if nextScreenView does not exists we can't calculate the duration (last event in session)if (nextScreenView) {const duration =- new Date(nextScreenView.event.created_at).getTime() -- new Date(item.event.created_at).getTime();+ new Date(nextScreenView.created_at).getTime() -+ new Date(item.created_at).getTime();const event = {- ...item.event,+ ...item,duration,};- event.properties.__duration_from = nextScreenView.event.id;- itemsToClickhouse.add({- ...item,- event,- });- // push last event in session if we have a session_end event+ event.properties.__duration_from = nextScreenView.id;+ toInsert.add(event);} else if (hasSessionEnd) {- itemsToClickhouse.add(item);+ // push last event in session if we have a session_end event+ toInsert.add(item);}});} // for of end@@ -158,8 +129,8 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {// This should not theoretically happen but if it does we should move them to stalledqueue.forEach((item) => {if (- !itemsToClickhouse.has(item) &&- new Date(item.event.created_at).getTime() <+ !toInsert.has(item) &&+ new Date(item.created_at).getTime() <new Date().getTime() - 1000 * 60 * 60 * 24) {itemsToStalled.add(item);@@ -169,43 +140,57 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {if (itemsToStalled.size > 0) {const multi = getRedisCache().multi();for (const item of itemsToStalled) {- multi.rpush(this.getKey('stalled'), JSON.stringify(item.event));+ multi.rpush(this.getKey('stalled'), JSON.stringify(item));}await multi.exec();}+ const toInsertArray = Array.from(toInsert);+ return {+ toInsert: toInsertArray,+ toKeep: queue.filter(+ (item) => !toInsertArray.find((i) => i.id === item.id),+ ),+ };+ }++ protected async insertIntoDB(items: BufferType[]): Promise<void> {await ch.insert({table: TABLE_NAMES.events,- values: Array.from(itemsToClickhouse).map((item) => item.event),+ values: items,format: 'JSONEachRow',});-- return [- ...Array.from(itemsToClickhouse).map((item) => item.index),- ...Array.from(itemsToStalled).map((item) => item.index),- ];- };+ }public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (callback,) => {- return this.getQueue(-1)- .then((queue) => {- return queue.filter(callback).map((item) => transformEvent(item.event));- })- .catch(() => {- return [];- });+ if (await this.waitForReleasedLock()) {+ return this.getQueue()+ .then((queue) => {+ return queue.filter(callback).map(transformEvent);+ })+ .catch(() => {+ return [];+ });+ }+ return [];};public find: Find<IClickhouseEvent, IServiceEvent> = async (callback) => {- return this.getQueue(-1)- .then((queue) => {- const match = queue.find(callback);- return match ? transformEvent(match.event) : null;- })- .catch(() => {- return null;- });+ if (await this.waitForReleasedLock()) {+ return this.getQueue(-1)+ .then((queue) => {+ const match = queue.find(callback);+ return match ? transformEvent(match) : null;+ })+ .catch(() => {+ return null;+ });+ }+ return null;};}++const sortOldestFirst = (a: IClickhouseEvent, b: IClickhouseEvent) =>+ new Date(a.created_at).getTime() - new Date(b.created_at).getTime();diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 208a1f0bb..63250871d 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -1,56 +1,80 @@-import { mergeDeepRight } from 'ramda';+import { groupBy, mergeDeepRight, prop } from 'ramda';import { toDots } from '@openpanel/common';import { getRedisCache } from '@openpanel/redis';+import { escape } from 'sqlstring';import { TABLE_NAMES, ch, chQuery } from '../clickhouse-client';+import { transformProfile } from '../services/profile.service';import type {IClickhouseProfile,IServiceProfile,} from '../services/profile.service';-import { transformProfile } from '../services/profile.service';-import type {- Find,- FindMany,- OnCompleted,- OnInsert,- ProcessQueue,- QueueItem,-} from './buffer';+import type { Find, FindMany } from './buffer';import { RedisBuffer } from './buffer';-export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {+const BATCH_SIZE = process.env.BATCH_SIZE_PROFILES+ ? Number.parseInt(process.env.BATCH_SIZE_PROFILES, 10)+ : 50;++type BufferType = IClickhouseProfile;+export class ProfileBuffer extends RedisBuffer<BufferType> {constructor() {- super({- table: TABLE_NAMES.profiles,- batchSize: 100,- disableAutoFlush: true,- });+ super(TABLE_NAMES.profiles, BATCH_SIZE);}- public onInsert?: OnInsert<IClickhouseProfile> | undefined;- public onCompleted?: OnCompleted<IClickhouseProfile> | undefined;-- public processQueue: ProcessQueue<IClickhouseProfile> = async (queue) => {- const cleanedQueue = this.combineQueueItems(queue);- const redisProfiles = await this.getCachedProfiles(cleanedQueue);+ // this will do a couple of things:+ // - we slice the queue to maxBufferSize since this queries have a limit on character count+ // - check redis cache for profiles+ // - fetch missing profiles from clickhouse+ // - merge the incoming profile with existing data+ protected async processItems(+ items: BufferType[],+ ): Promise<{ toInsert: BufferType[]; toKeep: BufferType[] }> {+ const queue = this.combineQueueItems(items);+ const slicedQueue = this.maxBufferSize+ ? queue.slice(0, this.maxBufferSize)+ : queue;+ const redisProfiles = await this.getCachedProfiles(slicedQueue);const dbProfiles = await this.fetchDbProfiles(- cleanedQueue.filter((_, index) => !redisProfiles[index]),+ slicedQueue.filter((_, index) => !redisProfiles[index]),);- const values = this.createProfileValues(- cleanedQueue,+ const toInsert = this.createProfileValues(+ slicedQueue,redisProfiles,dbProfiles,);- if (values.length > 0) {- await this.updateRedisCache(values);- await this.insertIntoClickhouse(values);+ if (toInsert.length > 0) {+ await this.updateRedisCache(toInsert);}- return queue.map((item) => item.index);- };+ return Promise.resolve({+ toInsert,+ toKeep: this.maxBufferSize ? queue.slice(this.maxBufferSize) : [],+ });+ }++ private combineQueueItems(queue: BufferType[]): BufferType[] {+ const itemsToClickhouse = new Map<string, BufferType>();++ queue.forEach((item) => {+ const key = item.project_id + item.id;+ const existing = itemsToClickhouse.get(key);+ itemsToClickhouse.set(key, mergeDeepRight(existing ?? {}, item));+ });++ return Array.from(itemsToClickhouse.values());+ }++ protected async insertIntoDB(items: BufferType[]): Promise<void> {+ await ch.insert({+ table: TABLE_NAMES.profiles,+ values: items,+ format: 'JSONEachRow',+ });+ }private matchPartialObject(full: any,@@ -77,27 +101,16 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {return true;}- private combineQueueItems(- queue: QueueItem<IClickhouseProfile>[],- ): QueueItem<IClickhouseProfile>[] {- const itemsToClickhouse = new Map<string, QueueItem<IClickhouseProfile>>();-- queue.forEach((item) => {- const key = item.event.project_id + item.event.id;- const existing = itemsToClickhouse.get(key);- itemsToClickhouse.set(key, mergeDeepRight(existing ?? {}, item));- });-- return Array.from(itemsToClickhouse.values());- }-private async getCachedProfiles(- cleanedQueue: QueueItem<IClickhouseProfile>[],+ queue: BufferType[],): Promise<(IClickhouseProfile | null)[]> {const redisCache = getRedisCache();- const keys = cleanedQueue.map(- (item) => `profile:${item.event.project_id}:${item.event.id}`,- );+ const keys = queue.map((item) => `profile:${item.project_id}:${item.id}`);++ if (keys.length === 0) {+ return [];+ }+const cachedProfiles = await redisCache.mget(...keys);return cachedProfiles.map((profile) => {try {@@ -109,34 +122,51 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {}private async fetchDbProfiles(- cleanedQueue: QueueItem<IClickhouseProfile>[],+ queue: IClickhouseProfile[],): Promise<IClickhouseProfile[]> {- if (cleanedQueue.length === 0) {+ if (queue.length === 0) {return [];}+ // const grouped = groupBy(prop('project_id'), queue);+ // const queries = Object.entries(grouped).map(([project_id, items]) => {+ // if (!items) {+ // return [];+ // }++ // return chQuery<IClickhouseProfile>(+ // `SELECT+ // *+ // FROM ${TABLE_NAMES.profiles}+ // WHERE+ // id IN (${items.map((item) => escape(item.id)).join(',')})+ // AND created_at > INTERVAL 12 MONTH+ // ORDER BY+ // created_at DESC`,+ // );+ // });+return await chQuery<IClickhouseProfile>(`SELECT*FROM ${TABLE_NAMES.profiles}WHERE- (id, project_id) IN (${cleanedQueue.map((item) => `('${item.event.id}', '${item.event.project_id}')`).join(',')})+ (project_id, id) IN (${queue.map((item) => `('${item.project_id}', '${item.id}')`).join(',')})ORDER BYcreated_at DESC`,);}private createProfileValues(- cleanedQueue: QueueItem<IClickhouseProfile>[],+ queue: IClickhouseProfile[],redisProfiles: (IClickhouseProfile | null)[],dbProfiles: IClickhouseProfile[],): IClickhouseProfile[] {- return cleanedQueue+ return queue.map((item, index) => {const cachedProfile = redisProfiles[index];const dbProfile = dbProfiles.find(- (p) =>- p.id === item.event.id && p.project_id === item.event.project_id,+ (p) => p.id === item.id && p.project_id === item.project_id,);const profile = cachedProfile || dbProfile;@@ -145,31 +175,33 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {this.matchPartialObject(profile,{- ...item.event,- properties: toDots(item.event.properties),+ ...item,+ properties: toDots(item.properties),},{ignore: ['created_at'],},)) {- console.log('Ignoring profile', item.event.id);+ this.logger.debug('No changes for profile', {+ profile,+ });return null;}return {- id: item.event.id,- first_name: item.event.first_name ?? profile?.first_name ?? '',- last_name: item.event.last_name ?? profile?.last_name ?? '',- email: item.event.email ?? profile?.email ?? '',- avatar: item.event.avatar ?? profile?.avatar ?? '',+ id: item.id,+ first_name: item.first_name ?? profile?.first_name ?? '',+ last_name: item.last_name ?? profile?.last_name ?? '',+ email: item.email ?? profile?.email ?? '',+ avatar: item.avatar ?? profile?.avatar ?? '',properties: toDots({...(profile?.properties ?? {}),- ...(item.event.properties ?? {}),+ ...(item.properties ?? {}),}),- project_id: item.event.project_id ?? profile?.project_id ?? '',- created_at: item.event.created_at ?? profile?.created_at ?? '',- is_external: item.event.is_external,+ project_id: item.project_id ?? profile?.project_id ?? '',+ created_at: item.created_at ?? profile?.created_at ?? '',+ is_external: item.is_external,};}).flatMap((item) => (item ? [item] : []));@@ -188,24 +220,12 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {await multi.exec();}- private async insertIntoClickhouse(- values: IClickhouseProfile[],- ): Promise<void> {- await ch.insert({- table: TABLE_NAMES.profiles,- values,- format: 'JSONEachRow',- });- }-public findMany: FindMany<IClickhouseProfile, IServiceProfile> = async (callback,) => {return this.getQueue(-1).then((queue) => {- return queue- .filter(callback)- .map((item) => transformProfile(item.event));+ return queue.filter(callback).map(transformProfile);}).catch(() => {return [];@@ -216,7 +236,7 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {return this.getQueue(-1).then((queue) => {const match = queue.find(callback);- return match ? transformProfile(match.event) : null;+ return match ? transformProfile(match) : null;}).catch(() => {return null;diff --git a/packages/db/src/buffers/trash/bot-buffer.ts b/packages/db/src/buffers/trash/bot-buffer.tsnew file mode 100644index 000000000..d30289872--- /dev/null+++ b/packages/db/src/buffers/trash/bot-buffer.ts@@ -0,0 +1,39 @@+import { TABLE_NAMES, ch } from '../../clickhouse-client';+import type { IClickhouseBotEvent } from '../../services/event.service';+import type {+ Find,+ FindMany,+ OnCompleted,+ OnInsert,+ ProcessQueue,+} from './buffer';+import { RedisBuffer } from './buffer';++export class BotBuffer extends RedisBuffer<IClickhouseBotEvent> {+ constructor() {+ super({+ table: TABLE_NAMES.events_bots,+ batchSize: 100,+ });+ }++ public onInsert?: OnInsert<IClickhouseBotEvent> | undefined;+ public onCompleted?: OnCompleted<IClickhouseBotEvent> | undefined;++ public processQueue: ProcessQueue<IClickhouseBotEvent> = async (queue) => {+ await ch.insert({+ table: TABLE_NAMES.events_bots,+ values: queue.map((item) => item.event),+ format: 'JSONEachRow',+ });+ return queue.map((item) => item.index);+ };++ public findMany: FindMany<IClickhouseBotEvent, IClickhouseBotEvent> = () => {+ return Promise.resolve([]);+ };++ public find: Find<IClickhouseBotEvent, IClickhouseBotEvent> = () => {+ return Promise.resolve(null);+ };+}diff --git a/packages/db/src/buffers/trash/buffer.ts b/packages/db/src/buffers/trash/buffer.tsnew file mode 100644index 000000000..55dbc8c0e--- /dev/null+++ b/packages/db/src/buffers/trash/buffer.ts@@ -0,0 +1,181 @@+import { createLogger } from '@openpanel/logger';+import { getRedisCache } from '@openpanel/redis';++export const DELETE = '__DELETE__';++export type QueueItem<T> = {+ event: T;+ index: number;+};++export type OnInsert<T> = (data: T) => unknown;++export type OnCompleted<T> =+ | ((data: T[]) => Promise<unknown[]>)+ | ((data: T[]) => unknown[]);++export type ProcessQueue<T> = (data: QueueItem<T>[]) => Promise<number[]>;++export type Find<T, R = unknown> = (+ callback: (item: QueueItem<T>) => boolean,+) => Promise<R | null>;++export type FindMany<T, R = unknown> = (+ callback: (item: QueueItem<T>) => boolean,+) => Promise<R[]>;++const getError = (e: unknown) => {+ if (e instanceof Error) {+ return [+ `Name: ${e.name}`,+ `Message: ${e.message}`,+ `Stack: ${e.stack}`,+ `Cause: ${e.cause ? String(e.cause) : ''}`,+ ].join('\n');+ }+ return 'Unknown error';+};++export abstract class RedisBuffer<T> {+ // constructor+ public prefix = 'op:buffer';+ public table: string;+ public batchSize?: number;+ public logger: ReturnType<typeof createLogger>;+ public disableAutoFlush?: boolean;++ // abstract methods+ public abstract onInsert?: OnInsert<T>;+ public abstract onCompleted?: OnCompleted<T>;+ public abstract processQueue: ProcessQueue<T>;+ public abstract find: Find<T, unknown>;+ public abstract findMany: FindMany<T, unknown>;++ constructor(options: {+ table: string;+ batchSize?: number;+ disableAutoFlush?: boolean;+ }) {+ this.table = options.table;+ this.batchSize = options.batchSize;+ this.disableAutoFlush = options.disableAutoFlush;+ this.logger = createLogger({ name: 'buffer' }).child({+ table: this.table,+ });+ }++ public getKey(name?: string) {+ const key = `${this.prefix}:${this.table}`;+ if (name) {+ return `${key}:${name}`;+ }+ return key;+ }++ public async insert(value: T) {+ this.onInsert?.(value);+ await getRedisCache().rpush(this.getKey(), JSON.stringify(value));++ const length = await getRedisCache().llen(this.getKey());+ this.logger.debug(+ `Inserted item into buffer ${this.table}. Current length: ${length}`,+ );++ if (!this.disableAutoFlush && this.batchSize && length >= this.batchSize) {+ this.logger.info(+ `Buffer ${this.table} reached batch size (${this.batchSize}). Flushing...`,+ );+ this.flush();+ }+ }++ public async flush() {+ try {+ const queue = await this.getQueue(this.batchSize || -1);++ if (queue.length === 0) {+ this.logger.debug(`Flush called on empty buffer ${this.table}`);+ return { count: 0, data: [] };+ }++ this.logger.info(+ `Flushing ${queue.length} items from buffer ${this.table}`,+ );++ try {+ const indexes = await this.processQueue(queue);+ await this.deleteIndexes(indexes);+ const data = indexes+ .map((index) => queue[index]?.event)+ .filter((event): event is T => event !== null);++ if (this.onCompleted) {+ const res = await this.onCompleted(data);+ this.logger.info(+ `Completed processing ${res.length} items from buffer ${this.table}`,+ );+ return { count: res.length, data: res };+ }++ this.logger.info(+ `Processed ${indexes.length} items from buffer ${this.table}`,+ );+ return { count: indexes.length, data: indexes };+ } catch (e) {+ this.logger.error(+ `Failed to process queue while flushing buffer ${this.table}:`,+ e,+ );+ const timestamp = new Date().getTime();+ await getRedisCache().hset(this.getKey(`failed:${timestamp}`), {+ error: getError(e),+ data: JSON.stringify(queue.map((item) => item.event)),+ retries: 0,+ });+ this.logger.warn(+ `Stored ${queue.length} failed items in ${this.getKey(`failed:${timestamp}`)}`,+ );+ }+ } catch (e) {+ this.logger.error(+ `Failed to get queue while flushing buffer ${this.table}:`,+ e,+ );+ }+ }++ public async deleteIndexes(indexes: number[]) {+ const multi = getRedisCache().multi();+ indexes.forEach((index) => {+ multi.lset(this.getKey(), index, DELETE);+ });+ multi.lrem(this.getKey(), 0, DELETE);+ await multi.exec();+ this.logger.debug(+ `Deleted ${indexes.length} items from buffer ${this.table}`,+ );+ }++ public async getQueue(limit: number): Promise<QueueItem<T>[]> {+ const queue = await getRedisCache().lrange(this.getKey(), 0, limit);+ const result = queue+ .map((item, index) => ({+ event: this.transformQueueItem(item),+ index,+ }))+ .filter((item): item is QueueItem<T> => item.event !== null);+ this.logger.debug(+ `Retrieved ${result.length} items from buffer ${this.table}`,+ );+ return result;+ }++ private transformQueueItem(item: string): T | null {+ try {+ return JSON.parse(item);+ } catch (e) {+ this.logger.warn(`Failed to parse item in buffer ${this.table}:`, e);+ return null;+ }+ }+}diff --git a/packages/db/src/buffers/trash/event-buffer.ts b/packages/db/src/buffers/trash/event-buffer.tsnew file mode 100644index 000000000..4616747b5--- /dev/null+++ b/packages/db/src/buffers/trash/event-buffer.ts@@ -0,0 +1,211 @@+import { groupBy, omit } from 'ramda';+import SuperJSON from 'superjson';++import { deepMergeObjects } from '@openpanel/common';+import { getRedisCache, getRedisPub } from '@openpanel/redis';++import { TABLE_NAMES, ch } from '../../clickhouse-client';+import { transformEvent } from '../../services/event.service';+import type {+ IClickhouseEvent,+ IServiceEvent,+} from '../../services/event.service';+import type {+ Find,+ FindMany,+ OnCompleted,+ OnInsert,+ ProcessQueue,+ QueueItem,+} from './buffer';+import { RedisBuffer } from './buffer';++const sortOldestFirst = (+ a: QueueItem<IClickhouseEvent>,+ b: QueueItem<IClickhouseEvent>,+) =>+ new Date(a.event.created_at).getTime() -+ new Date(b.event.created_at).getTime();++export class EventBuffer extends RedisBuffer<IClickhouseEvent> {+ constructor() {+ super({+ table: TABLE_NAMES.events,+ });+ }++ public onInsert?: OnInsert<IClickhouseEvent> | undefined = (event) => {+ getRedisPub().publish(+ 'event:received',+ SuperJSON.stringify(transformEvent(event)),+ );+ if (event.profile_id) {+ getRedisCache().set(+ `live:event:${event.project_id}:${event.profile_id}`,+ '',+ 'EX',+ 60 * 5,+ );+ }+ };++ public onCompleted?: OnCompleted<IClickhouseEvent> | undefined = (+ savedEvents,+ ) => {+ for (const event of savedEvents) {+ getRedisPub().publish(+ 'event:saved',+ SuperJSON.stringify(transformEvent(event)),+ );+ }++ return savedEvents.map((event) => event.id);+ };++ public processQueue: ProcessQueue<IClickhouseEvent> = async (queue) => {+ const itemsToClickhouse = new Set<QueueItem<IClickhouseEvent>>();+ const itemsToStalled = new Set<QueueItem<IClickhouseEvent>>();++ // Sort data by created_at+ // oldest first+ queue.sort(sortOldestFirst);++ // All events thats not a screen_view can be sent to clickhouse+ // We only need screen_views since we want to calculate the duration of each screen+ // To do this we need a minimum of 2 screen_views+ queue+ .filter(+ (item) =>+ item.event.name !== 'screen_view' || item.event.device === 'server',+ )+ .forEach((item) => {+ // Find the last event with data and merge it with the current event+ // We use profile_id here since this property can be set from backend as well+ const lastEventWithData = queue+ .slice(0, item.index)+ .findLast((lastEvent) => {+ return (+ lastEvent.event.project_id === item.event.project_id &&+ lastEvent.event.profile_id === item.event.profile_id &&+ lastEvent.event.path !== ''+ );+ });++ const event = deepMergeObjects<IClickhouseEvent>(+ omit(['properties', 'duration'], lastEventWithData?.event || {}),+ item.event,+ );++ if (lastEventWithData) {+ event.properties.__properties_from = lastEventWithData.event.id;+ }++ return itemsToClickhouse.add({+ ...item,+ event,+ });+ });++ // Group screen_view events by session_id+ const grouped = groupBy(+ (item) => item.event.session_id,+ queue.filter(+ (item) =>+ item.event.name === 'screen_view' && item.event.device !== 'server',+ ),+ );++ // Iterate over each group+ for (const [sessionId, screenViews] of Object.entries(grouped)) {+ if (sessionId === '' || !sessionId) {+ continue;+ }++ // If there is only one screen_view event we can send it back to redis since we can't calculate the duration+ const hasSessionEnd = queue.find(+ (item) =>+ item.event.name === 'session_end' &&+ item.event.session_id === sessionId,+ );++ screenViews+ ?.slice()+ .sort(sortOldestFirst)+ .forEach((item, index) => {+ const nextScreenView = screenViews[index + 1];+ // if nextScreenView does not exists we can't calculate the duration (last event in session)+ if (nextScreenView) {+ const duration =+ new Date(nextScreenView.event.created_at).getTime() -+ new Date(item.event.created_at).getTime();+ const event = {+ ...item.event,+ duration,+ };+ event.properties.__duration_from = nextScreenView.event.id;+ itemsToClickhouse.add({+ ...item,+ event,+ });+ // push last event in session if we have a session_end event+ } else if (hasSessionEnd) {+ itemsToClickhouse.add(item);+ }+ });+ } // for of end++ // Check if we have any events that has been in the queue for more than 24 hour+ // This should not theoretically happen but if it does we should move them to stalled+ queue.forEach((item) => {+ if (+ !itemsToClickhouse.has(item) &&+ new Date(item.event.created_at).getTime() <+ new Date().getTime() - 1000 * 60 * 60 * 24+ ) {+ itemsToStalled.add(item);+ }+ });++ if (itemsToStalled.size > 0) {+ const multi = getRedisCache().multi();+ for (const item of itemsToStalled) {+ multi.rpush(this.getKey('stalled'), JSON.stringify(item.event));+ }+ await multi.exec();+ }++ await ch.insert({+ table: TABLE_NAMES.events,+ values: Array.from(itemsToClickhouse).map((item) => item.event),+ format: 'JSONEachRow',+ });++ return [+ ...Array.from(itemsToClickhouse).map((item) => item.index),+ ...Array.from(itemsToStalled).map((item) => item.index),+ ];+ };++ public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (+ callback,+ ) => {+ return this.getQueue(-1)+ .then((queue) => {+ return queue.filter(callback).map((item) => transformEvent(item.event));+ })+ .catch(() => {+ return [];+ });+ };++ public find: Find<IClickhouseEvent, IServiceEvent> = async (callback) => {+ return this.getQueue(-1)+ .then((queue) => {+ const match = queue.find(callback);+ return match ? transformEvent(match.event) : null;+ })+ .catch(() => {+ return null;+ });+ };+}diff --git a/packages/db/src/buffers/trash/profile-buffer.ts b/packages/db/src/buffers/trash/profile-buffer.tsnew file mode 100644index 000000000..a4fe91b95--- /dev/null+++ b/packages/db/src/buffers/trash/profile-buffer.ts@@ -0,0 +1,225 @@+import { mergeDeepRight } from 'ramda';++import { toDots } from '@openpanel/common';+import { getRedisCache } from '@openpanel/redis';++import { TABLE_NAMES, ch, chQuery } from '../../clickhouse-client';+import type {+ IClickhouseProfile,+ IServiceProfile,+} from '../../services/profile.service';+import { transformProfile } from '../../services/profile.service';+import type {+ Find,+ FindMany,+ OnCompleted,+ OnInsert,+ ProcessQueue,+ QueueItem,+} from './buffer';+import { RedisBuffer } from './buffer';++export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {+ constructor() {+ super({+ table: TABLE_NAMES.profiles,+ batchSize: 100,+ disableAutoFlush: true,+ });+ }++ public onInsert?: OnInsert<IClickhouseProfile> | undefined;+ public onCompleted?: OnCompleted<IClickhouseProfile> | undefined;++ public processQueue: ProcessQueue<IClickhouseProfile> = async (queue) => {+ const cleanedQueue = this.combineQueueItems(queue);+ const redisProfiles = await this.getCachedProfiles(cleanedQueue);+ const dbProfiles = await this.fetchDbProfiles(+ cleanedQueue.filter((_, index) => !redisProfiles[index]),+ );++ const values = this.createProfileValues(+ cleanedQueue,+ redisProfiles,+ dbProfiles,+ );++ if (values.length > 0) {+ await this.updateRedisCache(values);+ await this.insertIntoClickhouse(values);+ }++ return queue.map((item) => item.index);+ };++ private matchPartialObject(+ full: any,+ partial: any,+ options: { ignore: string[] },+ ): boolean {+ if (typeof partial !== 'object' || partial === null) {+ return partial === full;+ }++ for (const key in partial) {+ if (options.ignore.includes(key)) {+ continue;+ }++ if (+ !(key in full) ||+ !this.matchPartialObject(full[key], partial[key], options)+ ) {+ return false;+ }+ }++ return true;+ }++ private combineQueueItems(+ queue: QueueItem<IClickhouseProfile>[],+ ): QueueItem<IClickhouseProfile>[] {+ const itemsToClickhouse = new Map<string, QueueItem<IClickhouseProfile>>();++ queue.forEach((item) => {+ const key = item.event.project_id + item.event.id;+ const existing = itemsToClickhouse.get(key);+ itemsToClickhouse.set(key, mergeDeepRight(existing ?? {}, item));+ });++ return Array.from(itemsToClickhouse.values());+ }++ private async getCachedProfiles(+ cleanedQueue: QueueItem<IClickhouseProfile>[],+ ): Promise<(IClickhouseProfile | null)[]> {+ const redisCache = getRedisCache();+ const keys = cleanedQueue.map(+ (item) => `profile:${item.event.project_id}:${item.event.id}`,+ );+ const cachedProfiles = await redisCache.mget(...keys);+ return cachedProfiles.map((profile) => {+ try {+ return profile ? JSON.parse(profile) : null;+ } catch (error) {+ return null;+ }+ });+ }++ private async fetchDbProfiles(+ cleanedQueue: QueueItem<IClickhouseProfile>[],+ ): Promise<IClickhouseProfile[]> {+ if (cleanedQueue.length === 0) {+ return [];+ }++ return await chQuery<IClickhouseProfile>(+ `SELECT+ *+ FROM ${TABLE_NAMES.profiles}+ WHERE+ (id, project_id) IN (${cleanedQueue.map((item) => `('${item.event.id}', '${item.event.project_id}')`).join(',')})+ ORDER BY+ created_at DESC`,+ );+ }++ private createProfileValues(+ cleanedQueue: QueueItem<IClickhouseProfile>[],+ redisProfiles: (IClickhouseProfile | null)[],+ dbProfiles: IClickhouseProfile[],+ ): IClickhouseProfile[] {+ return cleanedQueue+ .map((item, index) => {+ const cachedProfile = redisProfiles[index];+ const dbProfile = dbProfiles.find(+ (p) =>+ p.id === item.event.id && p.project_id === item.event.project_id,+ );+ const profile = cachedProfile || dbProfile;++ if (+ profile &&+ this.matchPartialObject(+ profile,+ {+ ...item.event,+ properties: toDots(item.event.properties),+ },+ {+ ignore: ['created_at'],+ },+ )+ ) {+ console.log('Ignoring profile', item.event.id);+ return null;+ }++ return {+ id: item.event.id,+ first_name: item.event.first_name ?? profile?.first_name ?? '',+ last_name: item.event.last_name ?? profile?.last_name ?? '',+ email: item.event.email ?? profile?.email ?? '',+ avatar: item.event.avatar ?? profile?.avatar ?? '',+ properties: toDots({+ ...(profile?.properties ?? {}),+ ...(item.event.properties ?? {}),+ }),+ project_id: item.event.project_id ?? profile?.project_id ?? '',+ created_at: item.event.created_at ?? profile?.created_at ?? '',+ is_external: item.event.is_external,+ };+ })+ .flatMap((item) => (item ? [item] : []));+ }++ private async updateRedisCache(values: IClickhouseProfile[]): Promise<void> {+ const redisCache = getRedisCache();+ const multi = redisCache.multi();+ values.forEach((value) => {+ multi.setex(+ `profile:${value.project_id}:${value.id}`,+ 60 * 30, // 30 minutes+ JSON.stringify(value),+ );+ });+ await multi.exec();+ }++ private async insertIntoClickhouse(+ values: IClickhouseProfile[],+ ): Promise<void> {+ await ch.insert({+ table: TABLE_NAMES.profiles,+ values,+ format: 'JSONEachRow',+ });+ }++ public findMany: FindMany<IClickhouseProfile, IServiceProfile> = async (+ callback,+ ) => {+ return this.getQueue(-1)+ .then((queue) => {+ return queue+ .filter(callback)+ .map((item) => transformProfile(item.event));+ })+ .catch(() => {+ return [];+ });+ };++ public find: Find<IClickhouseProfile, IServiceProfile> = async (callback) => {+ return this.getQueue(-1)+ .then((queue) => {+ const match = queue.find(callback);+ return match ? transformProfile(match.event) : null;+ })+ .catch(() => {+ return null;+ });+ };+}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 077bd632a..b0992b3f8 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -9,7 +9,6 @@ import type { IChartEventFilter } from '@openpanel/validation';import { botBuffer, eventBuffer } from '../buffers';import {TABLE_NAMES,- ch,chQuery,convertClickhouseDateToJs,formatClickhouseDate,@@ -339,7 +338,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {sdk_version: payload.sdkVersion ?? '',};- await eventBuffer.insert(event);+ await eventBuffer.add(event);return {document: event,@@ -562,7 +561,7 @@ export function createBotEvent({createdAt,path,}: IServiceCreateBotEventPayload) {- return botBuffer.insert({+ return botBuffer.add({id: uuid(),name,type,@@ -593,7 +592,7 @@ export async function getLastScreenViewFromProfileId({}const eventInBuffer = await eventBuffer.find(- (item) => item.event.profile_id === profileId,+ (item) => item.profile_id === profileId,);if (eventInBuffer) {diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 0a7c8d213..8e548fa25 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -222,7 +222,7 @@ export async function upsertProfile({projectId,isExternal,}: IServiceUpsertProfile) {- return profileBuffer.insert({+ return profileBuffer.add({id,first_name: firstName!,last_name: lastName!,diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex e24e97762..a6f8a9adf 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -28,12 +28,11 @@ export interface EventsQueuePayloadCreateEvent {type: 'createEvent';payload: Omit<IServiceEvent, 'id'>;}+type SessionEndRequired = 'sessionId' | 'deviceId' | 'profileId' | 'projectId';export interface EventsQueuePayloadCreateSessionEnd {type: 'createSessionEnd';- payload: Pick<- IServiceEvent,- 'deviceId' | 'sessionId' | 'profileId' | 'projectId'- >;+ payload: Partial<Omit<IServiceEvent, SessionEndRequired>> &+ Pick<IServiceEvent, SessionEndRequired>;}// TODO: Rename `EventsQueuePayloadCreateSessionEnd`@@ -66,6 +65,8 @@ export type CronQueuePayload =| CronQueuePayloadFlushProfiles| CronQueuePayloadPing;+export type CronQueueType = CronQueuePayload['type'];+export const eventsQueue = new Queue<EventsQueuePayload>('events', {connection: getRedisQueue(),defaultJobOptions: {