OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore(buffer): iron out the buffer issues
Carl-Gerhard Lindesvärd25f329a4cdd1
diff --git a/apps/api/scripts/mock-minimal.json b/apps/api/scripts/mock-minimal.jsonnew file mode 100644index 000000000..6eb5c6e25--- /dev/null+++ b/apps/api/scripts/mock-minimal.json@@ -0,0 +1,49 @@+[+ {+ "headers": {+ "openpanel-client-id": "5b679c47-9ec0-470a-8944-a9ab8f42b14f",+ "x-client-ip": "221.145.77.175",+ "user-agent": "Opera/13.66 (Macintosh; Intel Mac OS X 10.8.3 U; GV Presto/2.9.183 Version/11.00)",+ "origin": "https://classic-hovel.info"+ },+ "track": {+ "type": "track",+ "payload": {+ "name": "click_button"+ }+ }+ },+ {+ "headers": {+ "openpanel-client-id": "5b679c47-9ec0-470a-8944-a9ab8f42b14f",+ "x-client-ip": "221.145.77.175",+ "user-agent": "Opera/13.66 (Macintosh; Intel Mac OS X 10.8.3 U; GV Presto/2.9.183 Version/11.00)",+ "origin": "https://classic-hovel.info"+ },+ "track": {+ "type": "track",+ "payload": {+ "name": "click_button_2"+ }+ }+ },+ {+ "headers": {+ "openpanel-client-id": "5b679c47-9ec0-470a-8944-a9ab8f42b14f",+ "x-client-ip": "221.145.77.175",+ "user-agent": "Opera/13.66 (Macintosh; Intel Mac OS X 10.8.3 U; GV Presto/2.9.183 Version/11.00)",+ "origin": "https://classic-hovel.info"+ },+ "track": {+ "type": "track",+ "payload": {+ "name": "screen_view",+ "properties": {+ "__referrer": "https://www.google.com",+ "__path": "https://classic-hovel.info/beneficium-arcesso-quisquam",+ "__title": "Hic thesis laboriosam copiose admoveo sufficio."+ }+ }+ }+ }+]diff --git a/apps/api/scripts/mock.ts b/apps/api/scripts/mock.tsindex 248a592b0..38bfc18e6 100644--- a/apps/api/scripts/mock.ts+++ b/apps/api/scripts/mock.ts@@ -140,8 +140,9 @@ function scrambleEvents(events: Event[]) {return events.sort(() => Math.random() - 0.5);}-// Distribute events over 6 minutes-const SIX_MINUTES_MS = 3 * 60 * 1000;+// Distribute events over X minutes+const MINUTES = 3;+const SIX_MINUTES_MS = 1000 * 60 * MINUTES;const startTime = Date.now();let lastTriggeredIndex = 0;@@ -155,13 +156,14 @@ async function triggerEvents(file: string) {return;}- const eventsToTrigger = Math.floor(- generatedEvents.length * (elapsedTime / SIX_MINUTES_MS),+ const eventsToTrigger = Math.min(+ Math.ceil(generatedEvents.length * (elapsedTime / SIX_MINUTES_MS)),+ generatedEvents.length,);// Send events that haven't been triggered yetfor (let i = lastTriggeredIndex; i < eventsToTrigger; i++) {- console.log('asbout to send');+ console.log('about to send');const event = generatedEvents[i]!;try {@@ -171,7 +173,7 @@ async function triggerEvents(file: string) {console.error(`Failed to send event ${i + 1}:`, error);}console.log(- `sending ${event.track.payload.properties.__path} from user ${event.headers['user-agent']}`,+ `sending ${event.track.payload?.properties?.__path} from user ${event.headers['user-agent']}`,);}@@ -184,6 +186,8 @@ async function triggerEvents(file: string) {if (remainingEvents > 0) {setTimeout(() => triggerEvents(file), 50); // Check every 50ms+ } else {+ console.log('All events triggered.');}console.log(`Total events to trigger: ${generatedEvents.length}`);diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex b3189fcbe..1b82e4181 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -43,8 +43,8 @@ export async function postEvent(const locked = await getRedisCache().set(`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,'locked',- 'EX',- 5,+ 'PX',+ 950, // a bit under the delay below'NX',);@@ -72,7 +72,7 @@ export async function postEvent(// Prioritize 'screen_view' events by setting no delay// This ensures that session starts are created from 'screen_view' events// rather than other events, maintaining accurate session tracking- delay: request.body.name === 'screen_view' ? 0 : 1000,+ delay: request.body.name === 'screen_view' ? undefined : 1000,},);diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex f73a00748..e326cf06d 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -201,8 +201,8 @@ async function track({const locked = await getRedisCache().set(`request:priority:${currentDeviceId}-${previousDeviceId}:${isScreenView ? 'screen_view' : 'other'}`,'locked',- 'EX',- 5,+ 'PX',+ 950, // a bit under the delay below'NX',);@@ -228,7 +228,7 @@ async function track({// Prioritize 'screen_view' events by setting no delay// This ensures that session starts are created from 'screen_view' events// rather than other events, maintaining accurate session tracking- delay: payload.name === 'screen_view' ? 0 : 1000,+ delay: payload.name === 'screen_view' ? undefined : 1000,},);}diff --git a/apps/api/src/utils/parseIp.ts b/apps/api/src/utils/parseIp.tsindex 44b01af13..81e2e6e4f 100644--- a/apps/api/src/utils/parseIp.ts+++ b/apps/api/src/utils/parseIp.ts@@ -39,17 +39,22 @@ export async function parseIp(ip?: string): Promise<GeoLocation> {}try {- const geo = await fetch(`${process.env.GEO_IP_HOST}/${ip}`, {+ const res = await fetch(`${process.env.GEO_IP_HOST}/${ip}`, {signal: AbortSignal.timeout(2000),});- const res = (await geo.json()) as RemoteIpLookupResponse;++ if (!res.ok) {+ return geo;+ }++ const json = (await res.json()) as RemoteIpLookupResponse;return {- country: res.country,- city: res.city,- region: res.stateprov,- longitude: res.longitude,- latitude: res.latitude,+ country: json.country,+ city: json.city,+ region: json.stateprov,+ longitude: json.longitude,+ latitude: json.latitude,};} catch (error) {logger.error('Failed to fetch geo location for ip', { error });diff --git a/apps/api/src/utils/parseUserAgent.ts b/apps/api/src/utils/parseUserAgent.tsindex b7e8666d3..42ddc8bb2 100644--- a/apps/api/src/utils/parseUserAgent.ts+++ b/apps/api/src/utils/parseUserAgent.ts@@ -9,7 +9,9 @@ export function parseUserAgent(ua?: string | null) {if (!ua) return parsedServerUa;const res = new UAParser(ua).getResult();- if (isServer(ua)) return parsedServerUa;+ if (isServer(ua)) {+ return parsedServerUa;+ }return {os: res.os.name,@@ -77,7 +79,9 @@ function isServer(userAgent: string) {return true;}- return !!userAgent.match(/^([^\s]+\/[\d.]+\s*)+$/);+ // Matches user agents like "Go-http-client/1.0" or "Go Http Client/1.0"+ // It should just match the first name (with optional spaces) and version+ return !!userAgent.match(/^[^\/]+\/[\d.]+$/);}export function getDevice(ua: string) {diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex ee1b5f7d4..8fa4ca2f9 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -8,11 +8,19 @@ import {eventBuffer,getEvents,} from '@openpanel/db';+import { createLogger } from '@openpanel/logger';import type { EventsQueuePayloadCreateSessionEnd } from '@openpanel/queue';export async function createSessionEnd(job: Job<EventsQueuePayloadCreateSessionEnd>,) {+ const logger = createLogger({+ name: 'job:create-session-end',+ }).child({+ payload: job.data.payload,+ jobId: job.id,+ });+const payload = job.data.payload;const eventsInBuffer = await eventBuffer.findMany((item) => item.session_id === payload.sessionId,@@ -71,7 +79,7 @@ export async function createSessionEnd(throw new Error('Could not found session_start or any screen_view');}- job.log('Creating session_start since it was not found');+ logger.warn('Creating session_start since it was not found');sessionStart = {...firstScreenView,diff --git a/apps/worker/src/jobs/events.ts b/apps/worker/src/jobs/events.tsindex 549b77bc2..40fac0c12 100644--- a/apps/worker/src/jobs/events.ts+++ b/apps/worker/src/jobs/events.ts@@ -8,6 +8,7 @@ import type {EventsQueuePayloadIncomingEvent,} from '@openpanel/queue';+import { cacheable } from '@openpanel/redis';import { createSessionEnd } from './events.create-session-end';import { incomingEvent } from './events.incoming-event';@@ -30,11 +31,17 @@ export async function eventsJob(job: Job<EventsQueuePayload>) {}}-async function updateEventsCount(projectId: string) {+const getProjectEventsCount = cacheable(async function getProjectEventsCount(+ projectId: string,+) {const res = await chQuery<{ count: number }>(`SELECT count(*) as count FROM ${TABLE_NAMES.events} WHERE project_id = ${escape(projectId)}`,);- const count = res[0]?.count;+ return res[0]?.count;+}, 60 * 60);++async function updateEventsCount(projectId: string) {+ const count = await getProjectEventsCount(projectId);if (count) {await db.project.update({where: {diff --git a/packages/db/src/buffers/buffer.ts b/packages/db/src/buffers/buffer.tsindex 2035223c9..2de114234 100644--- a/packages/db/src/buffers/buffer.ts+++ b/packages/db/src/buffers/buffer.ts@@ -1,8 +1,10 @@import { v4 as uuidv4 } from 'uuid';+import { getSafeJson } from '@openpanel/common';import type { ILogger } from '@openpanel/logger';import { createLogger } from '@openpanel/logger';import { getRedisCache } from '@openpanel/redis';+import { pathOr } from 'ramda';export type Find<T, R = unknown> = (callback: (item: T) => boolean,@@ -42,7 +44,9 @@ export class RedisBuffer<T> {await getRedisCache().rpush(this.getKey(), JSON.stringify(item));const bufferSize = await getRedisCache().llen(this.getKey());- this.logger.debug(`Item added. Current size: ${bufferSize}`);+ this.logger.debug(+ `Item added (${pathOr('unknown', ['id'], item)}) Current size: ${bufferSize}`,+ );if (this.maxBufferSize && bufferSize >= this.maxBufferSize) {await this.tryFlush();@@ -66,6 +70,8 @@ export class RedisBuffer<T> {this.logger.debug('Lock acquired. Attempting to flush.');try {await this.flush();+ } catch (error) {+ this.logger.error('Failed to flush buffer', { error });} finally {await this.releaseLock(lockId);}@@ -117,18 +123,26 @@ export class RedisBuffer<T> {.exec();if (!result) {+ this.logger.error('No result from redis transaction', {+ result,+ });throw new Error('Redis transaction failed');}const lrange = result[0];if (!lrange || lrange[0] instanceof Error) {+ this.logger.error('Error from lrange', {+ result,+ });throw new Error('Redis transaction failed');}const items = lrange[1] as string[];- const parsedItems = items.map((item) => JSON.parse(item) as T);+ const parsedItems = items+ .map((item) => getSafeJson<T | null>(item) as T | null)+ .filter((item): item is T => item !== null);if (parsedItems.length === 0) {this.logger.debug('No items to flush');@@ -163,15 +177,15 @@ export class RedisBuffer<T> {} catch (error) {this.logger.error('Failed to process queue while flushing buffer', {error,- queueSize: items.length,+ queueSize: parsedItems.length,});- if (items.length > 0) {+ if (parsedItems.length > 0) {// Add back items to keepthis.logger.info('Adding all items back to buffer');await getRedisCache().lpush(this.getKey(),- ...items.map((item) => JSON.stringify(item)),+ ...parsedItems.map((item) => JSON.stringify(item)),);}}@@ -190,8 +204,15 @@ export class RedisBuffer<T> {}protected async getQueue(count?: number): Promise<T[]> {- const items = await getRedisCache().lrange(this.getKey(), 0, count ?? -1);- return items.map((item) => JSON.parse(item) as T);+ try {+ const items = await getRedisCache().lrange(this.getKey(), 0, count ?? -1);+ return items+ .map((item) => getSafeJson<T | null>(item) as T | null)+ .filter((item): item is T => item !== null);+ } catch (error) {+ this.logger.error('Failed to get queue', { error });+ return [];+ }}protected processItems(items: T[]): Promise<{ toInsert: T[]; toKeep: T[] }> {diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 3920c5898..4b836793b 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -4,7 +4,11 @@ import SuperJSON from 'superjson';import { deepMergeObjects } from '@openpanel/common';import { getRedisCache, getRedisPub } from '@openpanel/redis';-import { TABLE_NAMES, ch } from '../clickhouse-client';+import {+ TABLE_NAMES,+ ch,+ convertClickhouseDateToJs,+} from '../clickhouse-client';import { transformEvent } from '../services/event.service';import type {IClickhouseEvent,@@ -13,6 +17,8 @@ import type {import type { Find, FindMany } from './buffer';import { RedisBuffer } from './buffer';+const STALLED_QUEUE_TIMEOUT = 1000 * 60 * 60 * 24;+type BufferType = IClickhouseEvent;export class EventBuffer extends RedisBuffer<BufferType> {constructor() {@@ -46,8 +52,8 @@ export class EventBuffer extends RedisBuffer<BufferType> {protected async processItems(queue: BufferType[],): Promise<{ toInsert: BufferType[]; toKeep: BufferType[] }> {- const toInsert = new Set<BufferType>();- const itemsToStalled = new Set<BufferType>();+ const toInsert: BufferType[] = [];+ const toStalled: BufferType[] = [];// Sort data by created_at// oldest first@@ -85,7 +91,7 @@ export class EventBuffer extends RedisBuffer<BufferType> {event.properties.__properties_from = lastEventWithData.id;}- return toInsert.add(event);+ return toInsert.push(event);});// Group screen_view events by session_id@@ -125,10 +131,10 @@ export class EventBuffer extends RedisBuffer<BufferType> {},duration,};- toInsert.add(event);+ toInsert.push(event);} else if (hasSessionEnd) {// push last event in session if we have a session_end event- toInsert.add(item);+ toInsert.push(item);}});} // for of end@@ -137,28 +143,37 @@ export class EventBuffer extends RedisBuffer<BufferType> {// This should not theoretically happen but if it does we should move them to stalledqueue.forEach((item) => {if (- !toInsert.has(item) &&- new Date(item.created_at).getTime() <- new Date().getTime() - 1000 * 60 * 60 * 24+ !toInsert.find((i) => i.id === item.id) &&+ convertClickhouseDateToJs(item.created_at).getTime() <+ new Date().getTime() - STALLED_QUEUE_TIMEOUT) {- itemsToStalled.add(item);+ toStalled.push(item);}});- if (itemsToStalled.size > 0) {- const multi = getRedisCache().multi();- for (const item of itemsToStalled) {- multi.rpush(this.getKey('stalled'), JSON.stringify(item));+ if (toStalled.length > 0) {+ try {+ this.logger.info(`Pushing to stalled queue (${toStalled.length})`, {+ items: toStalled,+ count: toStalled.length,+ });+ await getRedisCache().rpush(+ this.getKey('stalled'),+ ...toStalled.map((item) => JSON.stringify(item)),+ );+ } catch (error) {+ toStalled.length = 0;+ this.logger.error('Failed to push to stalled queue', { error });}- await multi.exec();}- const toInsertArray = Array.from(toInsert);return {- toInsert: toInsertArray,- toKeep: queue.filter(- (item) => !toInsertArray.find((i) => i.id === item.id),- ),+ toInsert,+ toKeep: queue.filter((item) => {+ const willBeInserted = toInsert.find((i) => i.id === item.id);+ const willBeStalled = toStalled.find((i) => i.id === item.id);+ return willBeInserted === undefined && willBeStalled === undefined;+ }),};}diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex b0992b3f8..5d15dc5f8 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -276,9 +276,6 @@ export async function createEvent(payload: IServiceCreateEventPayload) {if (!payload.profileId) {payload.profileId = payload.deviceId;}- console.log(- `create event ${payload.name} for [deviceId]: ${payload.deviceId} [profileId]: ${payload.profileId} [projectId]: ${payload.projectId} [path]: ${payload.path}`,- );if (payload.profileId !== '') {await upsertProfile({