OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(buffer): ensure we don't have duplicate events
Carl-Gerhard Lindesvärd44721f84da36
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex ef0fde372..359269a61 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -41,11 +41,12 @@ export async function postEvent(const isScreenView = request.body.name === 'screen_view';// this will ensure that we don't have multiple events creating sessions+ const LOCK_DURATION = 1000;const locked = await getRedisCache().set(`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,'locked','PX',- 950, // a bit under the delay below+ LOCK_DURATION,'NX',);@@ -76,7 +77,7 @@ export async function postEvent(// Prioritize 'screen_view' events by setting no delay// This ensures that session starts are created from 'screen_view' events// rather than other events, maintaining accurate session tracking- delay: request.body.name === 'screen_view' ? undefined : 1000,+ delay: isScreenView ? undefined : LOCK_DURATION - 100,},);diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 02f154850..aa481d1fd 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -229,11 +229,12 @@ async function track({}) {const isScreenView = payload.name === 'screen_view';// this will ensure that we don't have multiple events creating sessions+ const LOCK_DURATION = 1000;const locked = await getRedisCache().set(`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,'locked','PX',- 950, // a bit under the delay below+ LOCK_DURATION,'NX',);@@ -264,7 +265,7 @@ async function track({// Prioritize 'screen_view' events by setting no delay// This ensures that session starts are created from 'screen_view' events// rather than other events, maintaining accurate session tracking- delay: isScreenView ? undefined : 1000,+ delay: isScreenView ? undefined : LOCK_DURATION - 100,},);}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 3dc825504..d615f423e 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -96,10 +96,10 @@ const startServer = async () => {global: false,});- fastify.addHook('preHandler', ipHook);- fastify.addHook('preHandler', timestampHook);- fastify.addHook('preHandler', fixHook);fastify.addHook('onRequest', requestIdHook);+ fastify.addHook('onRequest', timestampHook);+ fastify.addHook('onRequest', ipHook);+ fastify.addHook('onRequest', fixHook);fastify.addHook('onResponse', requestLoggingHook);fastify.register(compress, {diff --git a/packages/db/src/buffers/event-buffer-redis.ts b/packages/db/src/buffers/event-buffer-redis.tsindex f568ed76a..f209ac7b5 100644--- a/packages/db/src/buffers/event-buffer-redis.ts+++ b/packages/db/src/buffers/event-buffer-redis.ts@@ -393,13 +393,15 @@ return "OK"new Date(b.created_at || 0).getTime(),);+ const deduplicatedEvents = this.deduplicateEvents(eventsToClickhouse);+// (D) Insert events into ClickHouse in chunksthis.logger.info('Inserting events into ClickHouse', {- totalEvents: eventsToClickhouse.length,- chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),+ totalEvents: deduplicatedEvents.length,+ chunks: Math.ceil(deduplicatedEvents.length / this.chunkSize),});- for (const chunk of this.chunks(eventsToClickhouse, this.chunkSize)) {+ for (const chunk of this.chunks(deduplicatedEvents, this.chunkSize)) {await ch.insert({table: 'events',values: chunk,@@ -412,7 +414,7 @@ return "OK"// (E) Publish "saved" events.const pubMulti = getRedisPub().multi();- for (const event of eventsToClickhouse) {+ for (const event of deduplicatedEvents) {await publishEvent('events', 'saved', transformEvent(event), pubMulti);}await pubMulti.exec();@@ -448,6 +450,78 @@ return "OK"}}+ /**+ * Deduplicate events based on specified criteria+ *+ * - project_id+ * - name+ * - created_at+ * - profile_id or session_id or device_id+ */+ private deduplicateEvents(events: IClickhouseEvent[]): IClickhouseEvent[] {+ const { deduplicated, duplicates } = events.reduce<{+ seen: Map<string, IClickhouseEvent>;+ deduplicated: IClickhouseEvent[];+ duplicates: Array<{+ original: IClickhouseEvent;+ duplicate: IClickhouseEvent;+ }>;+ }>(+ (acc, event) => {+ // Create a unique key for the event based on deduplication criteria+ const key = [+ event.project_id,+ event.name,+ event.created_at,+ event.profile_id || '',+ event.session_id || '',+ event.device_id || '',+ ].join('|');++ const existing = acc.seen.get(key);+ if (existing) {+ // It's a duplicate+ acc.duplicates.push({ original: existing, duplicate: event });+ } else {+ // First occurrence+ acc.seen.set(key, event);+ acc.deduplicated.push(event);+ }++ return acc;+ },+ { seen: new Map(), deduplicated: [], duplicates: [] },+ );++ if (duplicates.length > 0) {+ this.logger.warn('Found duplicate events', {+ duplicateCount: duplicates.length,+ duplicates: duplicates.map(({ original, duplicate }) => ({+ original: {+ projectId: original.project_id,+ name: original.name,+ created_at: original.created_at,+ reqId: original.properties?.__reqId,+ sessionId: original.session_id,+ profileId: original.profile_id,+ deviceId: original.device_id,+ },+ duplicate: {+ projectId: duplicate.project_id,+ name: duplicate.name,+ created_at: duplicate.created_at,+ reqId: duplicate.properties?.__reqId,+ sessionId: duplicate.session_id,+ profileId: duplicate.profile_id,+ deviceId: duplicate.device_id,+ },+ })),+ });+ }++ return deduplicated;+ }+/*** Process a session's events.*