OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
improve(queue): how we handle incoming events and session ends
Carl-Gerhard Lindesvärd0d58a5bf0c70
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex 87b786068..6834891fa 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -55,15 +55,6 @@ export async function postEvent(return;}- const isScreenView = request.body.name === 'screen_view';- // this will ensure that we don't have multiple events creating sessions- const LOCK_DURATION = 1000;- const locked = await getLock(- `request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,- 'locked',- LOCK_DURATION,- );-await eventsQueue.add('event',{@@ -79,7 +70,6 @@ export async function postEvent(geo,currentDeviceId,previousDeviceId,- priority: locked,},},{@@ -88,10 +78,6 @@ export async function postEvent(type: 'exponential',delay: 200,},- // Prioritize 'screen_view' events by setting no delay- // This ensures that session starts are created from 'screen_view' events- // rather than other events, maintaining accurate session tracking- delay: isScreenView ? undefined : LOCK_DURATION - 100,},);diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex b42ca334f..ac926ee9a 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -284,15 +284,6 @@ async function track({timestamp: string;isTimestampFromThePast: boolean;}) {- const isScreenView = payload.name === 'screen_view';- // this will ensure that we don't have multiple events creating sessions- const LOCK_DURATION = 1000;- const locked = await getLock(- `request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,- 'locked',- LOCK_DURATION,- );-await eventsQueue.add('event',{@@ -308,7 +299,6 @@ async function track({geo,currentDeviceId,previousDeviceId,- priority: locked,},},{@@ -317,10 +307,6 @@ async function track({type: 'exponential',delay: 200,},- // Prioritize 'screen_view' events by setting no delay- // This ensures that session starts are created from 'screen_view' events- // rather than other events, maintaining accurate session tracking- delay: isScreenView ? undefined : LOCK_DURATION - 100,},);}diff --git a/apps/api/tsup.config.ts b/apps/api/tsup.config.tsindex 39de34408..d1bf3ea88 100644--- a/apps/api/tsup.config.ts+++ b/apps/api/tsup.config.ts@@ -11,7 +11,7 @@ const options: Options = {'@node-rs/argon2','bcrypt',],- ignoreWatch: ['../../**/{.git,node_modules}/**'],+ ignoreWatch: ['../../**/{.git,node_modules,dist}/**'],sourcemap: true,splitting: false,};diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex c3718603e..c8ca7f099 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -1,5 +1,4 @@import type { Job } from 'bullmq';-import { last } from 'ramda';import { logger as baseLogger } from '@/utils/logger';import { getTime } from '@openpanel/common';@@ -9,61 +8,56 @@ import {checkNotificationRulesForSessionEnd,createEvent,eventBuffer,+ formatClickhouseDate,getEvents,} from '@openpanel/db';-import type { ILogger } from '@openpanel/logger';import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';-async function getCompleteSession({+// Grabs session_start and screen_views + the last occured event+async function getNecessarySessionEvents({projectId,sessionId,- hoursInterval,+ createdAt,}: {projectId: string;sessionId: string;- hoursInterval: number;-}) {+ createdAt: Date;+}): Promise<ReturnType<typeof getEvents>> {const sql = `SELECT * FROM ${TABLE_NAMES.events}WHEREsession_id = '${sessionId}'AND project_id = '${projectId}'- AND created_at > now() - interval ${hoursInterval} HOUR- ORDER BY created_at DESC+ AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'+ AND (+ name IN ('screen_view', 'session_start')+ OR created_at = (+ SELECT MAX(created_at)+ FROM ${TABLE_NAMES.events}+ WHERE session_id = '${sessionId}'+ AND project_id = '${projectId}'+ AND created_at >= '${formatClickhouseDate(new Date(new Date(createdAt).getTime() - 1000 * 60 * 5))}'+ AND name NOT IN ('screen_view', 'session_start')+ )+ )+ ORDER BY created_at DESC;`;- return getEvents(sql);-}--async function getCompleteSessionWithSessionStart({- projectId,- sessionId,- logger,-}: {- projectId: string;- sessionId: string;- logger: ILogger;-}): Promise<ReturnType<typeof getEvents>> {- const intervals = [1, 6, 12, 24, 72];- let intervalIndex = 0;- for (const hoursInterval of intervals) {- const events = await getCompleteSession({+ const [lastScreenView, eventsInDb] = await Promise.all([+ eventBuffer.getLastScreenView({projectId,sessionId,- hoursInterval,- });-- if (events.find((event) => event.name === 'session_start')) {- return events;- }-- const nextHoursInterval = intervals[++intervalIndex];- if (nextHoursInterval) {- logger.warn(`Checking last ${nextHoursInterval} hours for session_start`);- }- }+ }),+ getEvents(sql),+ ]);- return [];+ // sort last inserted first+ return [lastScreenView, ...eventsInDb]+ .filter((event): event is IServiceEvent => !!event)+ .sort(+ (a, b) =>+ new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),+ );}export async function createSessionEnd(@@ -77,56 +71,27 @@ export async function createSessionEnd(const payload = job.data.payload;- const [lastScreenView, eventsInDb] = await Promise.all([- eventBuffer.getLastScreenView({- projectId: payload.projectId,- sessionId: payload.sessionId,- }),- getCompleteSessionWithSessionStart({- projectId: payload.projectId,- sessionId: payload.sessionId,- logger,- }),- ]);-- // sort last inserted first- const events = [lastScreenView, ...eventsInDb]- .filter((event): event is IServiceEvent => !!event)- .sort(- (a, b) =>- new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime(),- );-- const sessionDuration = events.reduce((acc, event) => {- return acc + event.duration;- }, 0);+ const events = await getNecessarySessionEvents({+ projectId: payload.projectId,+ sessionId: payload.sessionId,+ createdAt: payload.createdAt,+ });- let sessionStart = events.find((event) => event.name === 'session_start');- const lastEvent = events[0];+ const sessionStart = events.find((event) => event.name === 'session_start');const screenViews = events.filter((event) => event.name === 'screen_view');+ const lastEvent = events[0];if (!sessionStart) {- const firstScreenView = last(screenViews);-- if (!firstScreenView) {- throw new Error('Could not found session_start or any screen_view');- }-- logger.warn('Creating session_start since it was not found');-- sessionStart = {- ...firstScreenView,- name: 'session_start',- createdAt: new Date(getTime(firstScreenView.createdAt) - 100),- };-- await createEvent(sessionStart);+ throw new Error('No session_start found');}if (!lastEvent) {throw new Error('No last event found');}+ const sessionDuration =+ lastEvent.createdAt.getTime() - sessionStart.createdAt.getTime();+await checkNotificationRulesForSessionEnd(events);logger.info('Creating session_end', {@@ -135,7 +100,6 @@ export async function createSessionEnd(screenViews,sessionDuration,events,- lastScreenView: lastScreenView ? lastScreenView : 'none',});return createEvent({diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex 9825e3348..adc2272f1 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -1,9 +1,10 @@-import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';-import type { Job } from 'bullmq';-import { omit } from 'ramda';-import { logger as baseLogger } from '@/utils/logger';-import { createSessionEnd, getSessionEnd } from '@/utils/session-handler';+import { getReferrerWithQuery, parseReferrer } from '@/utils/parse-referrer';+import {+ createSessionEndJob,+ createSessionStart,+ getSessionEnd,+} from '@/utils/session-handler';import { isSameDomain, parsePath } from '@openpanel/common';import { parseUserAgent } from '@openpanel/common/server';import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';@@ -14,7 +15,11 @@ import {} from '@openpanel/db';import type { ILogger } from '@openpanel/logger';import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';+import { getLock } from '@openpanel/redis';+import { DelayedError, type Job } from 'bullmq';+import { omit } from 'ramda';import * as R from 'ramda';+import { v4 as uuid } from 'uuid';const GLOBAL_PROPERTIES = ['__path', '__referrer'];@@ -29,16 +34,18 @@ async function createEventAndNotify(jobData: Job<EventsQueuePayloadIncomingEvent>['data']['payload'],logger: ILogger,) {- await checkNotificationRulesForEvent(payload).catch((e) => {- logger.error('Error checking notification rules', { error: e });- });-logger.info('Creating event', { event: payload, jobData });-- return createEvent(payload);+ const [event] = await Promise.all([+ createEvent(payload),+ checkNotificationRulesForEvent(payload),+ ]);+ return event;}-export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {+export async function incomingEvent(+ job: Job<EventsQueuePayloadIncomingEvent>,+ token?: string,+) {const {geo,event: body,@@ -46,7 +53,6 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {projectId,currentDeviceId,previousDeviceId,- priority,} = job.data.payload;const properties = body.properties ?? {};const reqId = headers['request-id'] ?? 'unknown';@@ -131,32 +137,50 @@ export async function incomingEvent(job: Job<EventsQueuePayloadIncomingEvent>) {}const sessionEnd = await getSessionEnd({- priority,projectId,currentDeviceId,previousDeviceId,profileId,});- const lastScreenView = await eventBuffer.getLastScreenView({- projectId,- sessionId: sessionEnd.payload.sessionId,- });+ const lastScreenView = sessionEnd+ ? await eventBuffer.getLastScreenView({+ projectId,+ sessionId: sessionEnd.sessionId,+ })+ : null;const payload: IServiceCreateEventPayload = merge(baseEvent, {- deviceId: sessionEnd.payload.deviceId,- sessionId: sessionEnd.payload.sessionId,- referrer: sessionEnd.payload?.referrer,- referrerName: sessionEnd.payload?.referrerName,- referrerType: sessionEnd.payload?.referrerType,+ deviceId: sessionEnd?.deviceId ?? currentDeviceId,+ sessionId: sessionEnd?.sessionId ?? uuid(),+ referrer: sessionEnd?.referrer ?? baseEvent.referrer,+ referrerName: sessionEnd?.referrerName ?? baseEvent.referrerName,+ referrerType: sessionEnd?.referrerType ?? baseEvent.referrerType,// if the path is not set, use the last screen view pathpath: baseEvent.path || lastScreenView?.path || '',origin: baseEvent.origin || lastScreenView?.origin || '',} as Partial<IServiceCreateEventPayload>) as IServiceCreateEventPayload;- if (sessionEnd.notFound) {- await createSessionEnd({ payload });+ if (!sessionEnd) {+ // Too avoid several created sessions we just throw if a lock exists+ // This will than retry the job+ const lock = await getLock(+ `create-session-end:${currentDeviceId}`,+ 'locked',+ 1000,+ );++ if (!lock) {+ logger.warn('Move incoming event to delayed');+ await job.moveToDelayed(Date.now() + 50, token);+ throw new DelayedError();+ }+ await createSessionStart({ payload });}- return createEventAndNotify(payload, job.data.payload, logger);+ const event = await createEventAndNotify(payload, job.data.payload, logger);++ await createSessionEndJob({ payload });++ return event;}diff --git a/apps/worker/src/jobs/events.ts b/apps/worker/src/jobs/events.tsindex ec5738722..51298e297 100644--- a/apps/worker/src/jobs/events.ts+++ b/apps/worker/src/jobs/events.ts@@ -7,6 +7,9 @@ import type {import { incomingEvent } from './events.incoming-event';-export async function eventsJob(job: Job<EventsQueuePayload>) {- return await incomingEvent(job as Job<EventsQueuePayloadIncomingEvent>);+export async function eventsJob(job: Job<EventsQueuePayload>, token?: string) {+ return await incomingEvent(+ job as Job<EventsQueuePayloadIncomingEvent>,+ token,+ );}diff --git a/apps/worker/src/utils/session-handler.ts b/apps/worker/src/utils/session-handler.tsindex d09c7b68e..acb858f79 100644--- a/apps/worker/src/utils/session-handler.ts+++ b/apps/worker/src/utils/session-handler.ts@@ -3,22 +3,33 @@ import { type IServiceCreateEventPayload, createEvent } from '@openpanel/db';import {type EventsQueuePayloadCreateSessionEnd,sessionsQueue,- sessionsQueueEvents,} from '@openpanel/queue';import type { Job } from 'bullmq';-import { v4 as uuid } from 'uuid';+import { logger } from './logger';export const SESSION_TIMEOUT = 1000 * 60 * 30;const getSessionEndJobId = (projectId: string, deviceId: string) =>`sessionEnd:${projectId}:${deviceId}`;-export async function createSessionEnd({+export async function createSessionStart({payload,}: {payload: IServiceCreateEventPayload;}) {- await sessionsQueue.add(+ return createEvent({+ ...payload,+ name: 'session_start',+ createdAt: new Date(getTime(payload.createdAt) - 100),+ });+}++export async function createSessionEndJob({+ payload,+}: {+ payload: IServiceCreateEventPayload;+}) {+ return sessionsQueue.add('session',{type: 'createSessionEnd',@@ -34,12 +45,6 @@ export async function createSessionEnd({},},);-- await createEvent({- ...payload,- name: 'session_start',- createdAt: new Date(getTime(payload.createdAt) - 100),- });}export async function getSessionEnd({@@ -47,42 +52,33 @@ export async function getSessionEnd({currentDeviceId,previousDeviceId,profileId,- priority,}: {projectId: string;currentDeviceId: string;previousDeviceId: string;profileId: string;- priority: boolean;}) {const sessionEnd = await getSessionEndJob({- priority,projectId,currentDeviceId,previousDeviceId,});- const sessionEndPayload =- sessionEnd?.job.data.payload ||- ({- sessionId: uuid(),- deviceId: currentDeviceId,- profileId,- projectId,- } satisfies EventsQueuePayloadCreateSessionEnd['payload']);-if (sessionEnd) {- // If for some reason we have a session end job that is not a createSessionEnd job- if (sessionEnd.job.data.type !== 'createSessionEnd') {- throw new Error('Invalid session end job');+ // Hack: if session end job just got created, we want to give it a chance to complete+ // So the order is correct+ if (sessionEnd.job.timestamp > Date.now() - 50) {+ await new Promise((resolve) => setTimeout(resolve, 100));}- // If the profile_id is set and it's different from the device_id, we need to update the profile_id- if (- sessionEnd.job.data.payload.profileId !== profileId &&+ const existingSessionIsAnonymous =sessionEnd.job.data.payload.profileId ===- sessionEnd.job.data.payload.deviceId- ) {+ sessionEnd.job.data.payload.deviceId;++ const eventIsIdentified =+ sessionEnd.job.data.payload.profileId !== profileId;++ if (existingSessionIsAnonymous && eventIsIdentified) {await sessionEnd.job.updateData({...sessionEnd.job.data,payload: {@@ -93,25 +89,22 @@ export async function getSessionEnd({}await sessionEnd.job.changeDelay(SESSION_TIMEOUT);+ return sessionEnd.job.data.payload;}- return {- payload: sessionEndPayload,- notFound: !sessionEnd,- };+ return null;}export async function getSessionEndJob(args: {projectId: string;currentDeviceId: string;previousDeviceId: string;- priority: boolean;retryCount?: number;}): Promise<{deviceId: string;job: Job<EventsQueuePayloadCreateSessionEnd>;} | null> {- const { priority, retryCount = 0 } = args;+ const { retryCount = 0 } = args;if (retryCount >= 6) {throw new Error('Failed to get session end');@@ -125,46 +118,32 @@ export async function getSessionEndJob(args: {job: Job<EventsQueuePayloadCreateSessionEnd>;} | null> {const state = await job.getState();- if (state === 'delayed') {- return { deviceId, job };- }-- if (state === 'failed') {- await job.retry();- await job.waitUntilFinished(sessionsQueueEvents, 1000 * 10);- return getSessionEndJob({- ...args,- priority,+ if (state !== 'delayed') {+ logger.info(`[session-handler] Session end job is in "${state}" state`, {+ state,retryCount,+ jobTimestamp: new Date(job.timestamp).toISOString(),+ jobDelta: Date.now() - job.timestamp,+ jobId: job.id,+ reqId: job.data.payload.properties?.__reqId ?? 'unknown',+ payload: job.data.payload,});}- if (state === 'completed') {- await job.remove();- return getSessionEndJob({- ...args,- priority,- retryCount,- });+ if (state === 'delayed' || state === 'waiting') {+ return { deviceId, job };}- if (state === 'active' || state === 'waiting') {- await job.waitUntilFinished(sessionsQueueEvents, 1000 * 10);+ if (state === 'active') {+ await new Promise((resolve) => setTimeout(resolve, 100));return getSessionEndJob({...args,- priority,- retryCount,+ retryCount: retryCount + 1,});}- // Shady state here, just remove it and retry- if (state === 'unknown') {+ if (state === 'completed') {await job.remove();- return getSessionEndJob({- ...args,- priority,- retryCount,- });}return null;@@ -175,8 +154,7 @@ export async function getSessionEndJob(args: {getSessionEndJobId(args.projectId, args.currentDeviceId),);if (currentJob) {- const res = await handleJobStates(currentJob, args.currentDeviceId);- if (res) return res;+ return await handleJobStates(currentJob, args.currentDeviceId);}// Check previous device job@@ -184,15 +162,7 @@ export async function getSessionEndJob(args: {getSessionEndJobId(args.projectId, args.previousDeviceId),);if (previousJob) {- const res = await handleJobStates(previousJob, args.previousDeviceId);- if (res) return res;- }-- // If no job found and not priority, retry- if (!priority) {- const backoffDelay = 50 * 2 ** retryCount;- await new Promise((resolve) => setTimeout(resolve, backoffDelay));- return getSessionEndJob({ ...args, priority, retryCount: retryCount + 1 });+ return await handleJobStates(previousJob, args.previousDeviceId);}// Create sessiondiff --git a/apps/worker/tsup.config.ts b/apps/worker/tsup.config.tsindex 464d750a7..9e1c93dd9 100644--- a/apps/worker/tsup.config.ts+++ b/apps/worker/tsup.config.ts@@ -6,7 +6,7 @@ const options: Options = {entry: ['src/index.ts'],noExternal: [/^@openpanel\/.*$/u, /^@\/.*$/u],external: ['@hyperdx/node-opentelemetry', 'winston'],- ignoreWatch: ['../../**/{.git,node_modules}/**'],+ ignoreWatch: ['../../**/{.git,node_modules,dist}/**'],sourcemap: true,splitting: false,};diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/db/src/buffers/base-buffer.tsindex 01974f5b9..83aaca596 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/db/src/buffers/base-buffer.ts@@ -67,8 +67,6 @@ export class BaseBuffer {lockId,});}- } else {- this.logger.warn('Failed to acquire lock. Skipping flush.', { lockId });}}}diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/db/src/buffers/session-buffer.tsindex df791d6b1..70ebd48b2 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/db/src/buffers/session-buffer.ts@@ -58,14 +58,17 @@ export class SessionBuffer extends BaseBuffer {if (event.origin) {newSession.exit_origin = event.origin;}- newSession.duration =+ const duration =new Date(newSession.ended_at).getTime() -new Date(newSession.created_at).getTime();- if (newSession.duration < 0) {+ if (duration > 0) {+ newSession.duration = duration;+ } else {this.logger.warn('Session duration is negative', {+ duration,+ event,session: newSession,});- newSession.duration = 0;}newSession.properties = toDots({...(event.properties || {}),@@ -73,7 +76,7 @@ export class SessionBuffer extends BaseBuffer {});// newSession.revenue += event.properties?.__revenue ?? 0;- if (event.name === 'screen_view') {+ if (event.name === 'screen_view' && event.path) {newSession.screen_views.push(event.path);newSession.screen_view_count += 1;} else {@@ -161,8 +164,6 @@ export class SessionBuffer extends BaseBuffer {const sessions = await this.getSession(event);const [newSession] = sessions;- console.log(`Adding sessions ${sessions.length}`);-const multi = this.redis.multi();multi.set(`session:${newSession.id}`,diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 753ce6400..0c9eefdd1 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -1,8 +1,8 @@-import { path, assocPath, last, mergeDeepRight, pick, uniq } from 'ramda';+import { path, assocPath, last, mergeDeepRight } from 'ramda';import { escape } from 'sqlstring';import { v4 as uuid } from 'uuid';-import { toDots } from '@openpanel/common';+import { DateTime, toDots } from '@openpanel/common';import { cacheable, getCache } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';@@ -19,13 +19,8 @@ import type { EventMeta, Prisma } from '../prisma-client';import { db } from '../prisma-client';import { createSqlBuilder } from '../sql-builder';import { getEventFiltersWhereClause } from './chart.service';-import type { IClickhouseProfile, IServiceProfile } from './profile.service';-import {- getProfileById,- getProfiles,- transformProfile,- upsertProfile,-} from './profile.service';+import type { IServiceProfile } from './profile.service';+import { getProfileById, getProfiles, upsertProfile } from './profile.service';export type IImportedEvent = Omit<IClickhouseEvent,@@ -293,39 +288,6 @@ export async function createEvent(payload: IServiceCreateEventPayload) {payload.profileId = payload.deviceId;}- if (payload.profileId) {- const profile = {- id: String(payload.profileId),- isExternal: payload.profileId !== payload.deviceId,- projectId: payload.projectId,- properties: {- path: payload.path,- country: payload.country,- city: payload.city,- region: payload.region,- longitude: payload.longitude,- latitude: payload.latitude,- os: payload.os,- os_version: payload.osVersion,- browser: payload.browser,- browser_version: payload.browserVersion,- device: payload.device,- brand: payload.brand,- model: payload.model,- referrer: payload.referrer,- referrer_name: payload.referrerName,- referrer_type: payload.referrerType,- },- };-- if (- profile.isExternal ||- (profile.isExternal === false && payload.name === 'session_start')- ) {- await upsertProfile(profile, true);- }- }-const event: IClickhouseEvent = {id: uuid(),name: payload.name,@@ -336,7 +298,9 @@ export async function createEvent(payload: IServiceCreateEventPayload) {properties: toDots(payload.properties),path: payload.path ?? '',origin: payload.origin ?? '',- created_at: formatClickhouseDate(payload.createdAt),+ created_at: DateTime.fromJSDate(payload.createdAt)+ .setZone('UTC')+ .toFormat('yyyy-MM-dd HH:mm:ss.SSS'),country: payload.country ?? '',city: payload.city ?? '',region: payload.region ?? '',@@ -360,6 +324,39 @@ export async function createEvent(payload: IServiceCreateEventPayload) {await Promise.all([sessionBuffer.add(event), eventBuffer.add(event)]);+ if (payload.profileId) {+ const profile = {+ id: String(payload.profileId),+ isExternal: payload.profileId !== payload.deviceId,+ projectId: payload.projectId,+ properties: {+ path: payload.path,+ country: payload.country,+ city: payload.city,+ region: payload.region,+ longitude: payload.longitude,+ latitude: payload.latitude,+ os: payload.os,+ os_version: payload.osVersion,+ browser: payload.browser,+ browser_version: payload.browserVersion,+ device: payload.device,+ brand: payload.brand,+ model: payload.model,+ referrer: payload.referrer,+ referrer_name: payload.referrerName,+ referrer_type: payload.referrerType,+ },+ };++ if (+ profile.isExternal ||+ (profile.isExternal === false && payload.name === 'session_start')+ ) {+ await upsertProfile(profile, true);+ }+ }+return {document: event,};diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 668d2fc5e..672f02d88 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -22,14 +22,18 @@ export interface EventsQueuePayloadIncomingEvent {headers: Record<string, string | undefined>;currentDeviceId: string;previousDeviceId: string;- priority: boolean;};}export interface EventsQueuePayloadCreateEvent {type: 'createEvent';payload: Omit<IServiceEvent, 'id'>;}-type SessionEndRequired = 'sessionId' | 'deviceId' | 'profileId' | 'projectId';+type SessionEndRequired =+ | 'sessionId'+ | 'deviceId'+ | 'profileId'+ | 'projectId'+ | 'createdAt';export interface EventsQueuePayloadCreateSessionEnd {type: 'createSessionEnd';payload: Partial<Omit<IServiceEvent, SessionEndRequired>> &diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 67e944049..d02df427a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -4,6 +4,12 @@ settings:autoInstallPeers: trueexcludeLinksFromLockfile: false+catalogs:+ default:+ zod:+ specifier: ^3.24.2+ version: 3.24.2+importers:.:@@ -1066,6 +1072,40 @@ importers:specifier: ^5.2.2version: 5.6.3+ packages/fire:+ dependencies:+ '@faker-js/faker':+ specifier: ^9.0.1+ version: 9.0.1+ '@openpanel/common':+ specifier: workspace:*+ version: link:../common+ '@openpanel/db':+ specifier: workspace:*+ version: link:../db+ csv-parse:+ specifier: ^5.6.0+ version: 5.6.0+ date-fns:+ specifier: ^3.3.1+ version: 3.3.1+ devDependencies:+ '@openpanel/tsconfig':+ specifier: workspace:*+ version: link:../../tooling/typescript+ '@openpanel/validation':+ specifier: workspace:*+ version: link:../validation+ '@types/node':+ specifier: 20.14.8+ version: 20.14.8+ tsup:+ specifier: ^7.2.0+ version: 7.3.0([email protected])([email protected])+ typescript:+ specifier: ^5.2.2+ version: 5.6.3+packages/integrations:dependencies:'@slack/bolt':@@ -3155,6 +3195,7 @@ packages:'@faker-js/[email protected]':resolution: {integrity: sha512-4mDeYIgM3By7X6t5E6eYwLAa+2h4DeZDF7thhzIg6XB76jeEvMwadYAMCFJL/R4AnEBcAUO9+gL0vhy3s+qvZA==}engines: {node: '>=18.0.0', npm: '>=9.0.0'}+ deprecated: Please update to a newer version'@fastify/[email protected]':resolution: {integrity: sha512-/c/TW2bO/v9JeEgoD/g1G5GxGeCF1Hafdf79WPmUlgYiBXummY0oX3VVq4yFkKKVBKDNlaDUYoab7g38RpPqCQ==}@@ -7485,6 +7526,9 @@ packages:resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}+ resolution: {integrity: sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==}+resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==}@@ -19905,6 +19949,8 @@ snapshots:+ [email protected]: {}+dependencies:internmap: 1.0.1