OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): improve inserting events
Carl-Gerhard Lindesvärdefd24ca67af1
diff --git a/packages/common/src/object.ts b/packages/common/src/object.tsindex e012ed1b9..6ba6d816c 100644--- a/packages/common/src/object.ts+++ b/packages/common/src/object.ts@@ -4,7 +4,7 @@ import superjson from 'superjson';export function toDots(obj: Record<string, unknown>,path = ''-): Record<string, number | string | boolean> {+): Record<string, string> {return Object.entries(obj).reduce((acc, [key, value]) => {if (typeof value === 'object' && value !== null) {return {diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 8c05f0658..c236946a4 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -31,37 +31,130 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {public onCompleted?: OnCompleted<IClickhouseProfile> | undefined;public processQueue: ProcessQueue<IClickhouseProfile> = async (queue) => {+ const cleanedQueue = this.combineQueueItems(queue);+ const redisProfiles = await this.getCachedProfiles(cleanedQueue);+ const dbProfiles = await this.fetchDbProfiles(+ cleanedQueue.filter((_, index) => !redisProfiles[index])+ );++ const values = this.createProfileValues(+ cleanedQueue,+ redisProfiles,+ dbProfiles+ );++ if (values.length > 0) {+ await this.updateRedisCache(values);+ await this.insertIntoClickhouse(values);+ }++ return queue.map((item) => item.index);+ };++ private matchPartialObject(+ full: any,+ partial: any,+ options: { ignore: string[] }+ ): boolean {+ if (typeof partial !== 'object' || partial === null) {+ return partial === full;+ }++ for (const key in partial) {+ if (options.ignore.includes(key)) {+ continue;+ }++ if (+ !(key in full) ||+ !this.matchPartialObject(full[key], partial[key], options)+ ) {+ return false;+ }+ }++ return true;+ }++ private combineQueueItems(+ queue: QueueItem<IClickhouseProfile>[]+ ): QueueItem<IClickhouseProfile>[] {const itemsToClickhouse = new Map<string, QueueItem<IClickhouseProfile>>();- // Combine all writes to the same profilequeue.forEach((item) => {const key = item.event.project_id + item.event.id;const existing = itemsToClickhouse.get(key);- itemsToClickhouse.set(- item.event.project_id + item.event.id,- mergeDeepRight(existing ?? {}, item)- );+ itemsToClickhouse.set(key, mergeDeepRight(existing ?? {}, item));});- const cleanedQueue = Array.from(itemsToClickhouse.values());+ return Array.from(itemsToClickhouse.values());+ }- const profiles = await chQuery<IClickhouseProfile>(+ private async getCachedProfiles(+ cleanedQueue: QueueItem<IClickhouseProfile>[]+ ): Promise<(IClickhouseProfile | null)[]> {+ const redisCache = getRedisCache();+ const keys = cleanedQueue.map(+ (item) => `profile:${item.event.project_id}:${item.event.id}`+ );+ const cachedProfiles = await redisCache.mget(...keys);+ return cachedProfiles.map((profile) => {+ try {+ return profile ? JSON.parse(profile) : null;+ } catch (error) {+ return null;+ }+ });+ }++ private async fetchDbProfiles(+ cleanedQueue: QueueItem<IClickhouseProfile>[]+ ): Promise<IClickhouseProfile[]> {+ if (cleanedQueue.length === 0) {+ return [];+ }++ return await chQuery<IClickhouseProfile>(`SELECT*- FROM profiles+ FROM ${TABLE_NAMES.profiles}WHERE(id, project_id) IN (${cleanedQueue.map((item) => `('${item.event.id}', '${item.event.project_id}')`).join(',')})ORDER BYcreated_at DESC`);+ }- await ch.insert({- table: TABLE_NAMES.profiles,- values: cleanedQueue.map((item) => {- const profile = profiles.find(+ private createProfileValues(+ cleanedQueue: QueueItem<IClickhouseProfile>[],+ redisProfiles: (IClickhouseProfile | null)[],+ dbProfiles: IClickhouseProfile[]+ ): IClickhouseProfile[] {+ return cleanedQueue+ .map((item, index) => {+ const cachedProfile = redisProfiles[index];+ const dbProfile = dbProfiles.find((p) =>p.id === item.event.id && p.project_id === item.event.project_id);+ const profile = cachedProfile || dbProfile;++ if (+ profile &&+ this.matchPartialObject(+ profile,+ {+ ...item.event,+ properties: toDots(item.event.properties),+ },+ {+ ignore: ['created_at'],+ }+ )+ ) {+ console.log('Ignoring profile', item.event.id);+ return null;+ }return {id: item.event.id,@@ -74,14 +167,35 @@ export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {...(item.event.properties ?? {}),}),project_id: item.event.project_id ?? profile?.project_id ?? '',- created_at: new Date(),+ created_at: item.event.created_at ?? profile?.created_at ?? '',is_external: item.event.is_external,};- }),+ })+ .flatMap((item) => (item ? [item] : []));+ }++ private async updateRedisCache(values: IClickhouseProfile[]): Promise<void> {+ const redisCache = getRedisCache();+ const multi = redisCache.multi();+ values.forEach((value) => {+ multi.setex(+ `profile:${value.project_id}:${value.id}`,+ 60 * 30, // 30 minutes+ JSON.stringify(value)+ );+ });+ await multi.exec();+ }++ private async insertIntoClickhouse(+ values: IClickhouseProfile[]+ ): Promise<void> {+ await ch.insert({+ table: TABLE_NAMES.profiles,+ values,format: 'JSONEachRow',});- return queue.map((item) => item.index);- };+ }public findMany: FindMany<IClickhouseProfile, IServiceProfile> = async (callbackdiff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 428a05ca8..cde761aa7 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -622,7 +622,7 @@ export async function getTopPages({}) {const res = await chQuery<IServicePage>(`SELECT path, count(*) as count, project_id, first_value(created_at) as first_seen, max(properties['__title']) as title, origin- FROM events_v2+ FROM ${TABLE_NAMES.events}WHERE name = 'screen_view'AND project_id = ${escape(projectId)}AND created_at > now() - INTERVAL 30 DAYdiff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 7bd5d9058..e96d2d099 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -49,7 +49,7 @@ export async function getProfileById(id: string, projectId: string) {}const [profile] = await chQuery<IClickhouseProfile>(- `SELECT * FROM profiles WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`+ `SELECT * FROM ${TABLE_NAMES.profiles} WHERE id = ${escape(String(id))} AND project_id = ${escape(projectId)} ORDER BY created_at DESC LIMIT 1`);if (!profile) {