OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
feat: backfill profile id on events
Carl-Gerhard Lindesvärdee2756882426
diff --git a/apps/worker/src/boot-cron.ts b/apps/worker/src/boot-cron.tsindex b29a6a7c7..4a564e0d0 100644--- a/apps/worker/src/boot-cron.ts+++ b/apps/worker/src/boot-cron.ts@@ -58,6 +58,11 @@ export async function bootCron() {type: 'flushSessions',pattern: 1000 * 10,},+ {+ name: 'flush',+ type: 'flushProfileBackfill',+ pattern: 1000 * 30,+ },{name: 'insightsDaily',type: 'insightsDaily',diff --git a/apps/worker/src/jobs/cron.ts b/apps/worker/src/jobs/cron.tsindex 135e174df..2735f5658 100644--- a/apps/worker/src/jobs/cron.ts+++ b/apps/worker/src/jobs/cron.ts@@ -1,6 +1,6 @@import type { Job } from 'bullmq';-import { eventBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';+import { eventBuffer, profileBackfillBuffer, profileBuffer, sessionBuffer } from '@openpanel/db';import type { CronQueuePayload } from '@openpanel/queue';import { jobdeleteProjects } from './cron.delete-projects';@@ -23,6 +23,9 @@ export async function cronJob(job: Job<CronQueuePayload>) {case 'flushSessions': {return await sessionBuffer.tryFlush();}+ case 'flushProfileBackfill': {+ return await profileBackfillBuffer.tryFlush();+ }case 'ping': {return await ping();}diff --git a/apps/worker/src/jobs/events.create-session-end.ts b/apps/worker/src/jobs/events.create-session-end.tsindex d24e554eb..ea0eb2f03 100644--- a/apps/worker/src/jobs/events.create-session-end.ts+++ b/apps/worker/src/jobs/events.create-session-end.ts@@ -14,6 +14,7 @@ import {getEvents,getHasFunnelRules,getNotificationRulesByProjectId,+ profileBackfillBuffer,sessionBuffer,transformSessionToEvent,} from '@openpanel/db';@@ -34,9 +35,9 @@ async function getSessionEvents({endAt: Date;}): Promise<IServiceEvent[]> {const sql = `- SELECT * FROM ${TABLE_NAMES.events}- WHERE- session_id = '${sessionId}'+ SELECT * FROM ${TABLE_NAMES.events}+ WHERE+ session_id = '${sessionId}'AND project_id = '${projectId}'AND created_at BETWEEN '${formatClickhouseDate(startAt)}' AND '${formatClickhouseDate(endAt)}'ORDER BY created_at DESC LIMIT ${MAX_SESSION_EVENTS};@@ -91,6 +92,22 @@ export async function createSessionEnd(});}+ const profileId = session.profile_id || payload.profileId++ if (+ profileId !== session.device_id+ && process.env.EXPERIMENTAL_PROFILE_BACKFILL === '1'+ ) {+ const runOnProjects = process.env.EXPERIMENTAL_PROFILE_BACKFILL_PROJECTS?.split(',').filter(Boolean) ?? []+ if(runOnProjects.length === 0 || runOnProjects.includes(payload.projectId)) {+ await profileBackfillBuffer.add({+ projectId: payload.projectId,+ sessionId: payload.sessionId,+ profileId: profileId,+ });+ }+ }+// Create session end eventreturn createEvent({...payload,@@ -104,7 +121,7 @@ export async function createSessionEnd(createdAt: new Date(convertClickhouseDateToJs(session.ended_at).getTime() + 1000,),- profileId: session.profile_id || payload.profileId,+ profileId: profileId,});}diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex 17932ab95..5bafb88fc 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -1,5 +1,6 @@import { BotBuffer as BotBufferRedis } from './bot-buffer';import { EventBuffer as EventBufferRedis } from './event-buffer';+import { ProfileBackfillBuffer } from './profile-backfill-buffer';import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';import { SessionBuffer } from './session-buffer';@@ -7,3 +8,6 @@ export const eventBuffer = new EventBufferRedis();export const profileBuffer = new ProfileBufferRedis();export const botBuffer = new BotBufferRedis();export const sessionBuffer = new SessionBuffer();+export const profileBackfillBuffer = new ProfileBackfillBuffer();++export type { ProfileBackfillEntry } from './profile-backfill-buffer';diff --git a/packages/db/src/buffers/profile-backfill-buffer.ts b/packages/db/src/buffers/profile-backfill-buffer.tsnew file mode 100644index 000000000..175d38b6c--- /dev/null+++ b/packages/db/src/buffers/profile-backfill-buffer.ts@@ -0,0 +1,117 @@+import { getSafeJson } from '@openpanel/json';+import { type Redis, getRedisCache } from '@openpanel/redis';+import sqlstring from 'sqlstring';+import { TABLE_NAMES, ch, getReplicatedTableName } from '../clickhouse/client';+import { BaseBuffer } from './base-buffer';++export interface ProfileBackfillEntry {+ projectId: string;+ sessionId: string;+ profileId: string;+}++// Max session IDs per IN clause before we split into another query+const CHUNK_SIZE = 500;++export class ProfileBackfillBuffer extends BaseBuffer {+ private batchSize = process.env.PROFILE_BACKFILL_BUFFER_BATCH_SIZE+ ? Number.parseInt(process.env.PROFILE_BACKFILL_BUFFER_BATCH_SIZE, 10)+ : 1000;++ private readonly redisKey = 'profile-backfill-buffer';+ private redis: Redis;++ constructor() {+ super({+ name: 'profile-backfill',+ onFlush: async () => {+ await this.processBuffer();+ },+ });+ this.redis = getRedisCache();+ }++ async add(entry: ProfileBackfillEntry) {+ try {+ this.logger.info('Adding profile backfill entry', entry);+ await this.redis+ .multi()+ .rpush(this.redisKey, JSON.stringify(entry))+ .incr(this.bufferCounterKey)+ .exec();+ } catch (error) {+ this.logger.error('Failed to add profile backfill entry', { error });+ }+ }++ async processBuffer() {+ try {+ const raw = await this.redis.lrange(this.redisKey, 0, this.batchSize - 1);++ if (raw.length === 0) return;++ // Deduplicate by sessionId — last write wins (most recent profileId)+ const seen = new Map<string, ProfileBackfillEntry>();+ for (const r of raw) {+ const parsed = getSafeJson<ProfileBackfillEntry>(r);+ if (parsed) {+ seen.set(parsed.sessionId, parsed);+ }+ }+ const entries = Array.from(seen.values());++ const table = getReplicatedTableName(TABLE_NAMES.events);++ const chunks = this.chunks(entries, CHUNK_SIZE);+ let processedChunks = 0;++ for (const chunk of chunks) {+ const caseClause = chunk+ .map(({ sessionId, profileId }) => `WHEN ${sqlstring.escape(sessionId)} THEN ${sqlstring.escape(profileId)}`)+ .join('\n');+ const tupleList = chunk+ .map(({ projectId, sessionId }) => `(${sqlstring.escape(projectId)}, ${sqlstring.escape(sessionId)})`)+ .join(',');++ const query = `+ UPDATE ${table}+ SET profile_id = CASE session_id+ ${caseClause}+ END+ WHERE (project_id, session_id) IN (${tupleList})+ AND created_at > now() - INTERVAL 6 HOURS`;++ await ch.command({+ query,+ clickhouse_settings: {+ mutations_sync: '0',+ allow_experimental_lightweight_update: '1'+ },+ });++ processedChunks++;+ this.logger.info('Profile backfill chunk applied', {+ count: chunk.length,+ });+ }++ if (processedChunks === chunks.length) {+ await this.redis+ .multi()+ .ltrim(this.redisKey, raw.length, -1)+ .decrby(this.bufferCounterKey, raw.length)+ .exec();++ this.logger.info('Profile backfill buffer processed', {+ total: entries.length,+ });+ }+ } catch (error) {+ this.logger.error('Failed to process profile backfill buffer', { error });+ }+ }++ async getBufferSize() {+ return this.getBufferSizeWithCounter(() => this.redis.llen(this.redisKey));+ }+}diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 0260a3429..244d15a27 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -67,6 +67,10 @@ export const TABLE_NAMES = {* Non-clustered mode = self-hosted environments*/export function isClickhouseClustered(): boolean {+ if (process.env.CLICKHOUSE_CLUSTER === 'true' || process.env.CLICKHOUSE_CLUSTER === '1') {+ return true+ }+return !(process.env.SELF_HOSTED === 'true' || process.env.SELF_HOSTED === '1');diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 32b21a4cf..37ee7024d 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -119,11 +119,16 @@ export type CronQueuePayloadOnboarding = {type: 'onboarding';payload: undefined;};+export type CronQueuePayloadFlushProfileBackfill = {+ type: 'flushProfileBackfill';+ payload: undefined;+};export type CronQueuePayload =| CronQueuePayloadSalt| CronQueuePayloadFlushEvents| CronQueuePayloadFlushSessions| CronQueuePayloadFlushProfiles+ | CronQueuePayloadFlushProfileBackfill| CronQueuePayloadPing| CronQueuePayloadProject| CronQueuePayloadInsightsDaily