OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
improve(buffer): prep postgres buffer
Carl-Gerhard Lindesvärdff2dca42f6d1
diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex 42e427a8b..f2da99685 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -44,7 +44,7 @@ async function getCompleteSessionWithSessionStart({sessionId: string;logger: ILogger;}): Promise<ReturnType<typeof getEvents>> {- const intervals = [6, 12, 24, 72];+ const intervals = [1, 6, 12, 24, 72];let intervalIndex = 0;for (const hoursInterval of intervals) {const events = await getCompleteSession({@@ -76,6 +76,7 @@ export async function createSessionEnd(const payload = job.data.payload;+ // TODO: Get complete session from buffer to offload clickhouseconst [lastScreenView, eventsInDb] = await Promise.all([eventBuffer.getLastScreenView({projectId: payload.projectId,@@ -96,19 +97,6 @@ export async function createSessionEnd(new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),);- events.map((event, index) => {- job.log(- [- `Index: ${index}`,- `Event: ${event.name}`,- `Created: ${event.createdAt.toISOString()}`,- `DeviceId: ${event.deviceId}`,- `Profile: ${event.profileId}`,- `Path: ${event.path}`,- ].join('\n'),- );- });-const sessionDuration = events.reduce((acc, event) => {return acc + event.duration;}, 0);diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex a5810f79f..76ef3989f 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -70,7 +70,7 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {projectId,properties: omit(GLOBAL_PROPERTIES, {...properties,- user_agent: userAgent,+ __user_agent: userAgent,__hash: hash,__query: query,}),diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsnew file mode 100644index 000000000..01974f5b9--- /dev/null+++ b/packages/db/src/buffers/base-buffer.ts@@ -0,0 +1,74 @@+import { generateSecureId } from '@openpanel/common/server/id';+import { type ILogger, createLogger } from '@openpanel/logger';+import { getRedisCache } from '@openpanel/redis';++export class BaseBuffer {+ name: string;+ logger: ILogger;+ lockKey: string;+ lockTimeout = 60;+ onFlush: () => void;++ constructor(options: {+ name: string;+ onFlush: () => Promise<void>;+ }) {+ this.logger = createLogger({ name: options.name });+ this.name = options.name;+ this.lockKey = `lock:${this.name}`;+ this.onFlush = options.onFlush;+ }++ protected chunks<T>(items: T[], size: number) {+ const chunks = [];+ for (let i = 0; i < items.length; i += size) {+ chunks.push(items.slice(i, i + size));+ }+ return chunks;+ }++ private async releaseLock(lockId: string): Promise<void> {+ this.logger.debug('Releasing lock...');+ const script = `+ if redis.call("get", KEYS[1]) == ARGV[1] then+ return redis.call("del", KEYS[1])+ else+ return 0+ end+ `;+ await getRedisCache().eval(script, 1, this.lockKey, lockId);+ }++ async tryFlush() {+ const now = performance.now();+ const lockId = generateSecureId('lock');+ const acquired = await getRedisCache().set(+ this.lockKey,+ lockId,+ 'EX',+ this.lockTimeout,+ 'NX',+ );+ if (acquired === 'OK') {+ try {+ this.logger.info('Acquired lock. Processing buffer...', {+ lockId,+ });+ await this.onFlush();+ } catch (error) {+ this.logger.error('Failed to process buffer', {+ error,+ lockId,+ });+ } finally {+ await this.releaseLock(lockId);+ this.logger.info('Flush completed', {+ elapsed: performance.now() - now,+ lockId,+ });+ }+ } else {+ this.logger.warn('Failed to acquire lock. Skipping flush.', { lockId });+ }+ }+}diff --git a/packages/db/src/buffers/bot-buffer-psql.ts b/packages/db/src/buffers/bot-buffer-psql.tsindex d0a06d163..d9d67332f 100644--- a/packages/db/src/buffers/bot-buffer-psql.ts+++ b/packages/db/src/buffers/bot-buffer-psql.ts@@ -6,17 +6,20 @@ import { Prisma } from '@prisma/client';import { TABLE_NAMES, ch } from '../clickhouse-client';import { db } from '../prisma-client';import type { IClickhouseBotEvent } from '../services/event.service';+import { BaseBuffer } from './base-buffer';-export class BotBuffer {- private name = 'bot';- private lockKey = `lock:${this.name}`;- private logger: ILogger;- private lockTimeout = 60;+export class BotBuffer extends BaseBuffer {private daysToKeep = 1;private batchSize = 500;constructor() {- this.logger = createLogger({ name: this.name });+ super({+ name: 'bot',+ onFlush: async () => {+ await this.processBuffer();+ await this.tryCleanup();+ },+ });}async add(event: IClickhouseBotEvent) {@@ -44,43 +47,6 @@ export class BotBuffer {}}- private async releaseLock(lockId: string): Promise<void> {- this.logger.debug('Releasing lock...');- const script = `- if redis.call("get", KEYS[1]) == ARGV[1] then- return redis.call("del", KEYS[1])- else- return 0- end- `;- await getRedisCache().eval(script, 1, this.lockKey, lockId);- }-- async tryFlush() {- const lockId = generateSecureId('lock');- const acquired = await getRedisCache().set(- this.lockKey,- lockId,- 'EX',- this.lockTimeout,- 'NX',- );-- if (acquired === 'OK') {- try {- this.logger.info('Acquired lock. Processing buffer...');- await this.processBuffer();- await this.tryCleanup();- } catch (error) {- this.logger.error('Failed to process buffer', { error });- } finally {- await this.releaseLock(lockId);- }- } else {- this.logger.warn('Failed to acquire lock. Skipping flush.');- }- }-async processBuffer() {const eventsToProcess = await db.botEventBuffer.findMany({where: {diff --git a/packages/db/src/buffers/event-buffer-psql.ts b/packages/db/src/buffers/event-buffer-psql.tsindex 7bde8661d..0a94a4099 100644--- a/packages/db/src/buffers/event-buffer-psql.ts+++ b/packages/db/src/buffers/event-buffer-psql.ts@@ -10,17 +10,25 @@ import {type IServiceEvent,transformEvent,} from '../services/event.service';+import { BaseBuffer } from './base-buffer';-export class EventBuffer {- private name = 'event';- private logger: Logger;- private lockKey = `lock:${this.name}`;- private lockTimeout = 60;- private daysToKeep = 2;- private batchSize = 1000;+export class EventBuffer extends BaseBuffer {+ private daysToKeep = 3;+ private batchSize = process.env.EVENT_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)+ : 2000;+ private chunkSize = process.env.EVENT_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)+ : 1000;constructor() {- this.logger = createLogger({ name: this.name });+ super({+ name: 'event',+ onFlush: async () => {+ await this.processBuffer();+ await this.cleanup();+ },+ });}async add(event: IClickhouseEvent) {@@ -30,22 +38,23 @@ export class EventBuffer {projectId: event.project_id,eventId: event.id,name: event.name,- profileId: event.profile_id,- sessionId: event.session_id,+ profileId: event.profile_id || null,+ sessionId: event.session_id || null,payload: event,},});- // TODO: UNCOMMENT THIS!!!- // this.publishEvent('event:received', event);- // if (event.profile_id) {- // getRedisCache().set(- // `live:event:${event.project_id}:${event.profile_id}`,- // '',- // 'EX',- // 60 * 5,- // );- // }+ if (!process.env.TEST_NEW_BUFFER) {+ this.publishEvent('event:received', event);+ if (event.profile_id) {+ getRedisCache().set(+ `live:event:${event.project_id}:${event.profile_id}`,+ '',+ 'EX',+ 60 * 5,+ );+ }+ }} catch (error) {if (error instanceof Prisma.PrismaClientKnownRequestError) {if (error.code === 'P2002') {@@ -70,49 +79,19 @@ export class EventBuffer {}}- private async releaseLock(lockId: string): Promise<void> {- this.logger.debug('Releasing lock...');- const script = `- if redis.call("get", KEYS[1]) == ARGV[1] then- return redis.call("del", KEYS[1])- else- return 0- end- `;- await getRedisCache().eval(script, 1, this.lockKey, lockId);- }-- async tryFlush() {- const lockId = generateSecureId('lock');- const acquired = await getRedisCache().set(- this.lockKey,- lockId,- 'EX',- this.lockTimeout,- 'NX',- );- if (acquired === 'OK') {- try {- this.logger.info('Acquired lock. Processing buffer...');- await this.processBuffer();- await this.tryCleanup();- } catch (error) {- this.logger.error('Failed to process buffer', { error });- } finally {- await this.releaseLock(lockId);- }- } else {- this.logger.warn('Failed to acquire lock. Skipping flush.');- }- }-async processBuffer() {+ let now = performance.now();+ const timer: Record<string, number | undefined> = {+ fetchUnprocessedEvents: undefined,+ transformEvents: undefined,+ insertToClickhouse: undefined,+ markAsProcessed: undefined,+ };const eventsToProcess = await db.$queryRaw<IPrismaEventBuffer[]>`- WITH has_2_special AS (+ WITH has_more_than_2_events AS (SELECT "sessionId"FROM event_bufferWHERE "processedAt" IS NULL- AND name IN ('screen_view', 'session_start', 'session_end')GROUP BY "sessionId"HAVING COUNT(*) >= 2)@@ -120,30 +99,32 @@ export class EventBuffer {FROM event_buffer eWHERE e."processedAt" IS NULLAND (- -- 1) if the event name is NOT in the special set- e.name NOT IN ('screen_view', 'session_start', 'session_end')+ -- 1) all events except screen_view+ e.name != 'screen_view'OR- -- 2) if the event name IS in the special set AND- -- the session has >= 2 such unprocessed events- (- e.name IN ('screen_view', 'session_start', 'session_end')- AND e."sessionId" IN (SELECT "sessionId" FROM has_2_special)- )+ -- 2) if the session has >= 2 such unprocessed events+ e."sessionId" IN (SELECT "sessionId" FROM has_more_than_2_events))- ORDER BY e."createdAt" ASC -- or e.id, whichever "oldest first" logic you use+ ORDER BY e."createdAt" ASCLIMIT ${this.batchSize}`;+ timer.fetchUnprocessedEvents = performance.now() - now;+ now = performance.now();+const toInsert = eventsToProcess.reduce<IPrismaEventBuffer[]>((acc, event, index, list) => {+ // SCREEN VIEWif (event.name === 'screen_view') {- const nextScreenView = list.find(- (e, eIndex) =>- (e.name === 'screen_view' || e.name === 'session_end') &&- e.sessionId === event.sessionId &&- eIndex > index,- );-+ const nextScreenView = list+ .slice(index + 1)+ .find(+ (e) =>+ (e.name === 'screen_view' || e.name === 'session_end') &&+ e.sessionId === event.sessionId,+ );++ // Calculate durationif (nextScreenView && nextScreenView.name === 'screen_view') {event.payload.duration =new Date(nextScreenView.createdAt).getTime() -@@ -155,6 +136,20 @@ export class EventBuffer {if (!nextScreenView) {return acc;}+ } else {+ // OTHER EVENTS+ const currentScreenView = list+ .slice(0, index)+ .findLast(+ (e) =>+ e.name === 'screen_view' && e.sessionId === event.sessionId,+ );++ if (currentScreenView) {+ // Get path related info from the current screen view+ event.payload.path = currentScreenView.payload.path;+ event.payload.origin = currentScreenView.payload.origin;+ }}acc.push(event);@@ -164,17 +159,29 @@ export class EventBuffer {[],);+ timer.transformEvents = performance.now() - now;+ now = performance.now();+if (toInsert.length > 0) {- await ch.insert({- table: 'events',- values: toInsert.map((e) => e.payload),- format: 'JSONEachRow',- });+ const events = toInsert.map((e) => e.payload);+ for (const chunk of this.chunks(events, this.chunkSize)) {+ await ch.insert({+ table: 'events',+ values: chunk,+ format: 'JSONEachRow',+ });+ }++ timer.insertToClickhouse = performance.now() - now;+ now = performance.now();for (const event of toInsert) {this.publishEvent('event:saved', event.payload);}+ timer.markAsProcessed = performance.now() - now;+ now = performance.now();+await db.eventBuffer.updateMany({where: {id: {@@ -186,8 +193,11 @@ export class EventBuffer {},});+ timer.markAsProcessed = performance.now() - now;+this.logger.info('Processed events', {count: toInsert.length,+ timer,});}}diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex 69909f097..ee4924f49 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -1,7 +1,16 @@import { BotBuffer } from './bot-buffer';+import { BotBuffer as NewBotBuffer } from './bot-buffer-psql';import { EventBuffer } from './event-buffer';+import { EventBuffer as NewEventBuffer } from './event-buffer-psql';import { ProfileBuffer } from './profile-buffer';+import { ProfileBuffer as NewProfileBuffer } from './profile-buffer-psql';-export const eventBuffer = new EventBuffer();-export const profileBuffer = new ProfileBuffer();-export const botBuffer = new BotBuffer();+export const eventBuffer = process.env.USE_NEW_BUFFER+ ? new NewEventBuffer()+ : new EventBuffer();+export const profileBuffer = process.env.USE_NEW_BUFFER+ ? new NewProfileBuffer()+ : new ProfileBuffer();+export const botBuffer = process.env.USE_NEW_BUFFER+ ? new NewBotBuffer()+ : new BotBuffer();diff --git a/packages/db/src/buffers/profile-buffer-psql.ts b/packages/db/src/buffers/profile-buffer-psql.tsindex 3046fdba5..0747d9eac 100644--- a/packages/db/src/buffers/profile-buffer-psql.ts+++ b/packages/db/src/buffers/profile-buffer-psql.ts@@ -7,16 +7,25 @@ import { mergeDeepRight } from 'ramda';import { TABLE_NAMES, ch, chQuery } from '../clickhouse-client';import { db } from '../prisma-client';import type { IClickhouseProfile } from '../services/profile.service';+import { BaseBuffer } from './base-buffer';-export class ProfileBuffer {- private name = 'profile';- private logger: Logger;- private lockKey = `lock:${this.name}`;- private lockTimeout = 60;+export class ProfileBuffer extends BaseBuffer {private daysToKeep = 30;+ private batchSize = process.env.EVENT_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)+ : 2000;+ private chunkSize = process.env.EVENT_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)+ : 1000;constructor() {- this.logger = createLogger({ name: this.name });+ super({+ name: 'profile',+ onFlush: async () => {+ await this.processBuffer();+ await this.tryCleanup();+ },+ });}private generateChecksum(profile: IClickhouseProfile): string {@@ -27,7 +36,6 @@ export class ProfileBuffer {async add(profile: IClickhouseProfile) {try {const checksum = this.generateChecksum(profile);-// Check if we have this exact profile in bufferconst existingProfile = await db.profileBuffer.findFirst({where: {@@ -75,8 +83,10 @@ export class ProfileBuffer {id: existingProfile.id,},data: {+ checksum: this.generateChecksum(mergedProfile),payload: mergedProfile,updatedAt: new Date(),+ processedAt: null, // unsure this will get processed (race condition)},});} else {@@ -110,43 +120,6 @@ export class ProfileBuffer {return result[0] || null;}- private async releaseLock(lockId: string): Promise<void> {- this.logger.debug('Releasing lock...');- const script = `- if redis.call("get", KEYS[1]) == ARGV[1] then- return redis.call("del", KEYS[1])- else- return 0- end- `;- await getRedisCache().eval(script, 1, this.lockKey, lockId);- }-- async tryFlush() {- const lockId = generateSecureId('lock');- const acquired = await getRedisCache().set(- this.lockKey,- lockId,- 'EX',- this.lockTimeout,- 'NX',- );-- if (acquired === 'OK') {- try {- this.logger.info('Acquired lock. Processing buffer...');- await this.processBuffer();- await this.tryCleanup();- } catch (error) {- this.logger.error('Failed to process buffer', { error });- } finally {- await this.releaseLock(lockId);- }- } else {- this.logger.warn('Failed to acquire lock. Skipping flush.');- }- }-async processBuffer() {const profilesToProcess = await db.profileBuffer.findMany({where: {@@ -155,6 +128,7 @@ export class ProfileBuffer {orderBy: {createdAt: 'asc',},+ take: this.batchSize,});if (profilesToProcess.length > 0) {@@ -163,11 +137,13 @@ export class ProfileBuffer {return profile;});- await ch.insert({- table: TABLE_NAMES.profiles,- values: toInsert,- format: 'JSONEachRow',- });+ for (const chunk of this.chunks(profilesToProcess, this.chunkSize)) {+ await ch.insert({+ table: TABLE_NAMES.profiles,+ values: chunk,+ format: 'JSONEachRow',+ });+ }await db.profileBuffer.updateMany({where: {diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 62e0e53bf..4a14b0ec5 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -273,11 +273,11 @@ export async function getEvents(}export async function createEvent(payload: IServiceCreateEventPayload) {- if (!payload.profileId) {+ if (!payload.profileId && payload.deviceId) {payload.profileId = payload.deviceId;}- if (payload.profileId !== '') {+ if (payload.profileId) {await upsertProfile({id: String(payload.profileId),isExternal: payload.profileId !== payload.deviceId,@@ -310,7 +310,7 @@ export async function createEvent(payload: IServiceCreateEventPayload) {profile_id: payload.profileId ? String(payload.profileId) : '',project_id: payload.projectId,session_id: payload.sessionId,- properties: toDots(omit(['_path'], payload.properties)),+ properties: toDots(payload.properties),path: payload.path ?? '',origin: payload.origin ?? '',created_at: formatClickhouseDate(payload.createdAt),