OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(buffer): move to redis again 😅 but better
Carl-Gerhard Lindesvärdec8aaf24fb6b
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex 9989c5c46..4497bebf0 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -71,11 +71,7 @@ register.registerMetric(name: `buffer_${eventBuffer.name}_count`,help: 'Number of unprocessed events',async collect() {- const metric = await db.eventBuffer.count({- where: {- processedAt: null,- },- });+ const metric = await eventBuffer.getBufferSize();this.set(metric);},}),@@ -86,11 +82,7 @@ register.registerMetric(name: `buffer_${profileBuffer.name}_count`,help: 'Number of unprocessed profiles',async collect() {- const metric = await db.profileBuffer.count({- where: {- processedAt: null,- },- });+ const metric = await profileBuffer.getBufferSize();this.set(metric);},}),@@ -101,11 +93,7 @@ register.registerMetric(name: `buffer_${botBuffer.name}_count`,help: 'Number of unprocessed bot events',async collect() {- const metric = await db.botEventBuffer.count({- where: {- processedAt: null,- },- });+ const metric = await botBuffer.getBufferSize();this.set(metric);},}),diff --git a/packages/db/src/buffers/bot-buffer-psql.ts b/packages/db/src/buffers/bot-buffer-psql.tsindex db0d9416e..40a5e135b 100644--- a/packages/db/src/buffers/bot-buffer-psql.ts+++ b/packages/db/src/buffers/bot-buffer-psql.ts@@ -105,4 +105,12 @@ export class BotBuffer extends BaseBuffer {this.logger.info('Cleaned up old bot events', { deleted: deleted.count });}++ public async getBufferSize() {+ return db.botEventBuffer.count({+ where: {+ processedAt: null,+ },+ });+ }}diff --git a/packages/db/src/buffers/bot-buffer-redis.ts b/packages/db/src/buffers/bot-buffer-redis.tsnew file mode 100644index 000000000..fa8edd70c--- /dev/null+++ b/packages/db/src/buffers/bot-buffer-redis.ts@@ -0,0 +1,77 @@+import { type Redis, getRedisCache, runEvery } from '@openpanel/redis';++import { getSafeJson } from '@openpanel/common';+import { TABLE_NAMES, ch } from '../clickhouse-client';+import type { IClickhouseBotEvent } from '../services/event.service';+import { BaseBuffer } from './base-buffer';++export class BotBuffer extends BaseBuffer {+ private batchSize = process.env.BOT_BUFFER_BATCH_SIZE+ ? Number.parseInt(process.env.BOT_BUFFER_BATCH_SIZE, 10)+ : 1000;++ private readonly redisKey = 'bot-events-buffer';+ private redis: Redis;+ constructor() {+ super({+ name: 'bot',+ onFlush: async () => {+ await this.processBuffer();+ },+ });+ this.redis = getRedisCache();+ }++ async add(event: IClickhouseBotEvent) {+ try {+ // Add event to Redis list+ await this.redis.rpush(this.redisKey, JSON.stringify(event));++ // Check buffer length+ const bufferLength = await this.redis.llen(this.redisKey);++ if (bufferLength >= this.batchSize) {+ await this.tryFlush();+ }+ } catch (error) {+ this.logger.error('Failed to add bot event', { error });+ }+ }++ async processBuffer() {+ try {+ // Get events from the start without removing them+ const events = await this.redis.lrange(+ this.redisKey,+ 0,+ this.batchSize - 1,+ );++ if (events.length === 0) return;++ const parsedEvents = events.map((e) =>+ getSafeJson<IClickhouseBotEvent>(e),+ );++ // Insert to ClickHouse+ await ch.insert({+ table: TABLE_NAMES.events_bots,+ values: parsedEvents,+ format: 'JSONEachRow',+ });++ // Only remove events after successful insert+ await this.redis.ltrim(this.redisKey, events.length, -1);++ this.logger.info('Processed bot events', {+ count: events.length,+ });+ } catch (error) {+ this.logger.error('Failed to process buffer', { error });+ }+ }++ async getBufferSize() {+ return getRedisCache().llen(this.redisKey);+ }+}diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/db/src/buffers/bot-buffer.tsdeleted file mode 100644index a96e421c4..000000000--- a/packages/db/src/buffers/bot-buffer.ts+++ /dev/null@@ -1,28 +0,0 @@-import { TABLE_NAMES, ch } from '../clickhouse-client';-import type { IClickhouseBotEvent } from '../services/event.service';-import { BotBuffer as NewBotBuffer } from './bot-buffer-psql';-import { RedisBuffer } from './buffer';--const testNewBotBuffer = new NewBotBuffer();--type BufferType = IClickhouseBotEvent;-export class BotBuffer extends RedisBuffer<BufferType> {- constructor() {- super('events_bots', 500);- }-- async add(event: BufferType) {- await super.add(event);- if (process.env.TEST_NEW_BUFFER) {- await testNewBotBuffer.add(event);- }- }-- protected async insertIntoDB(items: BufferType[]): Promise<void> {- await ch.insert({- table: TABLE_NAMES.events_bots,- values: items,- format: 'JSONEachRow',- });- }-}diff --git a/packages/db/src/buffers/event-buffer-psql.ts b/packages/db/src/buffers/event-buffer-psql.tsindex 45bca8d0d..a1b9f69b9 100644--- a/packages/db/src/buffers/event-buffer-psql.ts+++ b/packages/db/src/buffers/event-buffer-psql.ts@@ -296,4 +296,12 @@ export class EventBuffer extends BaseBuffer {}) {return `session:last_screen_view:${projectId}:${profileId}`;}++ async getBufferSize() {+ return db.eventBuffer.count({+ where: {+ processedAt: null,+ },+ });+ }}diff --git a/packages/db/src/buffers/event-buffer-redis.ts b/packages/db/src/buffers/event-buffer-redis.tsnew file mode 100644index 000000000..1bbfc1689--- /dev/null+++ b/packages/db/src/buffers/event-buffer-redis.ts@@ -0,0 +1,674 @@+import { getSafeJson, setSuperJson } from '@openpanel/common';+import {+ type Redis,+ getRedisCache,+ getRedisPub,+ runEvery,+} from '@openpanel/redis';+import { ch } from '../clickhouse-client';+import {+ type IClickhouseEvent,+ type IServiceEvent,+ transformEvent,+} from '../services/event.service';+import { BaseBuffer } from './base-buffer';++/**+ *+ * Usuful redis commands:+ * ---------------------+ *+ * Add empty session+ * ZADD event_buffer:sessions_sorted 1710831600000 "test_empty_session"+ *+ * Get session events+ * LRANGE event_buffer:session:test_empty_session 0 -1+ *+ * Get session events count+ * LLEN event_buffer:session:test_empty_session+ *+ * Get regular queue events+ * LRANGE event_buffer:regular_queue 0 -1+ *+ * Get regular queue count+ * LLEN event_buffer:regular_queue+ *+ */++export class EventBuffer extends BaseBuffer {+ // Configurable limits+ private daysToKeep = process.env.EVENT_BUFFER_DAYS_TO_KEEP+ ? Number.parseFloat(process.env.EVENT_BUFFER_DAYS_TO_KEEP)+ : 3;+ private batchSize = process.env.EVENT_BUFFER_BATCH_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_BATCH_SIZE, 10)+ : 4000;+ private chunkSize = process.env.EVENT_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.EVENT_BUFFER_CHUNK_SIZE, 10)+ : 1000;+ private updatePendingSessionsBatchSize = process.env+ .EVENT_BUFFER_UPDATE_PENDING_SESSIONS_BATCH_SIZE+ ? Number.parseInt(+ process.env.EVENT_BUFFER_UPDATE_PENDING_SESSIONS_BATCH_SIZE,+ 10,+ )+ : 1000;++ private sessionEvents = ['screen_view', 'session_end'];++ // LIST - Stores events without sessions+ private regularQueueKey = 'event_buffer:regular_queue';++ // SORTED SET - Tracks all active session IDs with their timestamps+ private sessionSortedKey = 'event_buffer:sessions_sorted'; // sorted set of session IDs++ private readonly sessionKeyPrefix = 'event_buffer:session:';+ // LIST - Stores events for a given session+ private getSessionKey(sessionId: string) {+ return `${this.sessionKeyPrefix}${sessionId}`;+ }+ /**+ * Lua script that loops through sessions and returns a JSON-encoded list of+ * session objects (sessionId and events). It stops once a total number of events+ * >= batchSize is reached. It also cleans up any empty sessions.+ */+ private readonly processSessionsScript = `+local sessionSortedKey = KEYS[1]+local sessionPrefix = KEYS[2]+local batchSize = tonumber(ARGV[1])+local minEvents = tonumber(ARGV[2]) -- New parameter for minimum events++local result = {}+local sessionsToRemove = {}+local sessionIds = redis.call('ZRANGE', sessionSortedKey, 0, -1)+local resultIndex = 1+local totalEvents = 0++for i, sessionId in ipairs(sessionIds) do+ local sessionKey = sessionPrefix .. sessionId+ local events = redis.call('LRANGE', sessionKey, 0, -1)++ if #events == 0 then+ table.insert(sessionsToRemove, sessionId)+ elseif #events >= minEvents then -- Use the parameter instead of hardcoded value+ result[resultIndex] = { sessionId = sessionId, events = events }+ resultIndex = resultIndex + 1+ totalEvents = totalEvents + #events+ if totalEvents >= batchSize then+ break+ end+ end+end++if #sessionsToRemove > 0 then+ redis.call('ZREM', sessionSortedKey, unpack(sessionsToRemove))+end++return cjson.encode(result)+`;++ /**+ * New atomic Lua script to update a session's list with pending events.+ * Instead of doing a separate DEL and RPUSH (which leaves a race condition),+ * this script will:+ * 1. Remove the first `snapshotCount` items from the session list.+ * 2. Re-insert the pending events (provided as additional arguments)+ * at the head (using LPUSH in reverse order to preserve order).+ *+ * KEYS[1] = session key+ * ARGV[1] = snapshotCount (number of events that were present in our snapshot)+ * ARGV[2] = pendingCount (number of pending events)+ * ARGV[3..(2+pendingCount)] = the pending event strings+ */+ private readonly updateSessionScript = `+local snapshotCount = tonumber(ARGV[1])+local pendingCount = tonumber(ARGV[2])+local sessionKey = KEYS[1]++-- Trim the list to remove the processed (snapshot) events.+redis.call("LTRIM", sessionKey, snapshotCount, -1)++-- Re-insert the pending events at the head in their original order.+for i = pendingCount, 1, -1 do+ redis.call("LPUSH", sessionKey, ARGV[i+2])+end++return redis.call("LLEN", sessionKey)+`;++ /**+ * Lua script that processes a batch of session updates in a single call.+ * Format of updates: [sessionKey1, snapshotCount1, pendingCount1, pending1...., sessionKey2, ...]+ */+ private readonly batchUpdateSessionsScript = `+local i = 1+while i <= #ARGV do+ local sessionKey = ARGV[i]+ local snapshotCount = tonumber(ARGV[i + 1])+ local pendingCount = tonumber(ARGV[i + 2])++ -- Trim the list to remove processed events+ redis.call("LTRIM", sessionKey, snapshotCount, -1)++ -- Re-insert pending events at the head in original order+ if pendingCount > 0 then+ local pendingEvents = {}+ for j = 1, pendingCount do+ table.insert(pendingEvents, ARGV[i + 2 + j])+ end+ redis.call("LPUSH", sessionKey, unpack(pendingEvents))+ end++ i = i + 3 + pendingCount+end+return "OK"+`;++ constructor() {+ super({+ name: 'event',+ onFlush: async () => {+ await this.processBuffer();+ await this.tryCleanup();+ },+ });+ }++ bulkAdd(events: IClickhouseEvent[]) {+ const redis = getRedisCache();+ const multi = redis.multi();+ for (const event of events) {+ this.add(event, multi);+ }+ return multi.exec();+ }++ /**+ * Add an event into Redis.+ * Combines multiple Redis operations into a single MULTI command.+ */+ async add(event: IClickhouseEvent, _multi?: ReturnType<Redis['multi']>) {+ try {+ const redis = getRedisCache();+ const eventJson = JSON.stringify(event);+ const multi = _multi || redis.multi();++ if (event.session_id && this.sessionEvents.includes(event.name)) {+ if (event.name === 'screen_view') {+ multi.set(+ this.getLastEventKey({+ projectId: event.project_id,+ profileId: event.profile_id,+ }),+ eventJson,+ 'EX',+ 60 * 31,+ );+ } else if (event.name === 'session_end') {+ multi.del(+ this.getLastEventKey({+ projectId: event.project_id,+ profileId: event.profile_id,+ }),+ );+ }++ const sessionKey = this.getSessionKey(event.session_id);+ const score = new Date(event.created_at || Date.now()).getTime();++ multi+ .rpush(sessionKey, eventJson)+ .zadd(this.sessionSortedKey, 'NX', score, event.session_id);+ } else {+ // All other events go to regularQueue queue+ multi.rpush(this.regularQueueKey, eventJson);+ }++ if (event.profile_id) {+ multi.set(+ `live:event:${event.project_id}:${event.profile_id}`,+ '',+ 'EX',+ 60 * 5,+ );+ }++ if (!_multi) {+ await multi.exec();+ }+ await this.publishEvent('event:received', event);+ } catch (error) {+ this.logger.error('Failed to add event to Redis buffer', { error });+ }+ }++ private async publishEvent(+ channel: string,+ event: IClickhouseEvent,+ multi?: ReturnType<Redis['multi']>,+ ) {+ try {+ await (multi || getRedisPub()).publish(+ channel,+ setSuperJson(+ transformEvent(event) as unknown as Record<string, unknown>,+ ),+ );+ } catch (error) {+ this.logger.warn('Failed to publish event', { error });+ }+ }++ private async getEligableSessions({ minEventsInSession = 2 }) {+ const sessionsSorted = await getRedisCache().eval(+ this.processSessionsScript,+ 2, // number of KEYS+ this.sessionSortedKey,+ this.sessionKeyPrefix,+ (this.batchSize / 2).toString(),+ minEventsInSession.toString(),+ );++ // (A) Process session events using the Lua script.+ const parsed = getSafeJson<+ Array<{+ sessionId: string;+ events: string[];+ }>+ >(sessionsSorted as string);++ const sessions: Record<string, IClickhouseEvent[]> = {};+ if (!parsed) {+ return sessions;+ }++ for (const session of parsed) {+ // Might be redundant check+ if (session.events.length > 1) {+ sessions[session.sessionId] = session.events+ .map((e) => getSafeJson<IClickhouseEvent>(e))+ .filter((e): e is IClickhouseEvent => e !== null);+ }+ }++ return sessions;+ }++ /**+ * Process the Redis buffer.+ *+ * 1. Fetch events from two sources in parallel:+ * - Pick events from regular queue (batchSize / 2)+ * - Pick events from sessions (batchSize / 2).+ * This only have screen_view and session_end events+ *+ * 2. Process session events:+ * - For screen_view events, calculate duration if next event exists+ * - Last screen_view of each session remains pending+ * - All other events are marked for flushing+ *+ * 3. Process regular queue events:+ * - Inherit path/origin from last screen_view of same session if exists+ *+ * 4. Insert all flushable events into ClickHouse in chunks and publish notifications+ *+ * 5. Clean up processed events:+ * - For regular queue: LTRIM processed events+ * - For sessions: Update lists atomically via Lua script, preserving pending events+ */+ async processBuffer() {+ const redis = getRedisCache();+ const eventsToClickhouse: IClickhouseEvent[] = [];+ const pendingUpdates: Array<{+ sessionId: string;+ snapshotCount: number;+ pending: IClickhouseEvent[];+ }> = [];+ const timer = {+ fetchUnprocessedEvents: 0,+ processSessionEvents: 0,+ processRegularQueueEvents: 0,+ insertEvents: 0,+ updatePendingSessions: 0,+ };++ try {+ let now = performance.now();+ const [sessions, regularQueueEvents] = await Promise.all([+ // (A) Fetch session events+ this.getEligableSessions({ minEventsInSession: 2 }),+ // (B) Fetch no-session events+ redis.lrange(this.regularQueueKey, 0, this.batchSize / 2 - 1),+ ]);++ timer.fetchUnprocessedEvents = performance.now() - now;+ now = performance.now();++ for (const [sessionId, sessionEvents] of Object.entries(sessions)) {+ const { flush, pending } = this.processSessionEvents(sessionEvents);++ if (flush.length > 0) {+ eventsToClickhouse.push(...flush);+ }++ pendingUpdates.push({+ sessionId,+ snapshotCount: sessionEvents.length,+ pending,+ });+ }++ timer.processSessionEvents = performance.now() - now;+ now = performance.now();++ // (C) Sort events by creation time.+ eventsToClickhouse.sort(+ (a, b) =>+ new Date(a.created_at || 0).getTime() -+ new Date(b.created_at || 0).getTime(),+ );++ // (B) Process no-session events+ for (const eventStr of regularQueueEvents) {+ const event = getSafeJson<IClickhouseEvent>(eventStr);+ if (event) {+ const sessionEvents = sessions[event.session_id] || [];+ const screenView = sessionEvents.findLast((sessionEvent) => {+ const isScreenView = sessionEvent.name === 'screen_view';+ const isBeforeEvent =+ new Date(sessionEvent.created_at).getTime() <+ new Date(event.created_at).getTime();++ return isScreenView && isBeforeEvent;+ });++ if (screenView) {+ event.path = screenView.path;+ event.origin = screenView.origin;+ event.properties.__inherit_from = screenView.id;+ }++ eventsToClickhouse.push(event);+ }+ }++ timer.processRegularQueueEvents = performance.now() - now;+ now = performance.now();++ if (eventsToClickhouse.length === 0) {+ this.logger.debug('No events to process');+ return;+ }++ // (C) Sort events by creation time.+ eventsToClickhouse.sort(+ (a, b) =>+ new Date(a.created_at || 0).getTime() -+ new Date(b.created_at || 0).getTime(),+ );++ // (D) Insert events into ClickHouse in chunks+ this.logger.info('Inserting events into ClickHouse', {+ totalEvents: eventsToClickhouse.length,+ chunks: Math.ceil(eventsToClickhouse.length / this.chunkSize),+ });++ for (const chunk of this.chunks(eventsToClickhouse, this.chunkSize)) {+ await ch.insert({+ table: 'events',+ values: chunk,+ format: 'JSONEachRow',+ });+ }++ timer.insertEvents = performance.now() - now;+ now = performance.now();++ // (E) Publish "saved" events.+ const pubMulti = getRedisPub().multi();+ for (const event of eventsToClickhouse) {+ await this.publishEvent('event:saved', event, pubMulti);+ }+ await pubMulti.exec();++ // (F) Only after successful processing, update Redis+ const multi = redis.multi();++ // Clean up no-session events+ if (regularQueueEvents.length > 0) {+ multi.ltrim(this.regularQueueKey, regularQueueEvents.length, -1);+ }++ await multi.exec();++ // Process pending sessions in batches+ await this.processPendingSessionsInBatches(redis, pendingUpdates);++ timer.updatePendingSessions = performance.now() - now;++ this.logger.info('Processed events from Redis buffer', {+ batchSize: this.batchSize,+ eventsToClickhouse: eventsToClickhouse.length,+ pendingSessionUpdates: pendingUpdates.length,+ sessionEvents: Object.entries(sessions).reduce(+ (acc, [sId, events]) => acc + events.length,+ 0,+ ),+ regularEvents: regularQueueEvents.length,+ timer,+ });+ } catch (error) {+ this.logger.error('Error processing Redis buffer', { error });+ }+ }++ /**+ * Process a session's events.+ *+ * For each event in the session (in order):+ * - If it is a screen_view, look for a subsequent event (screen_view or session_end)+ * to calculate its duration. If found, flush it; if not, leave it pending.+ *+ * Returns an object with two arrays:+ * flush: events to be sent to ClickHouse.+ * pending: events that remain in the Redis session list.+ */+ private processSessionEvents(events: IClickhouseEvent[]): {+ flush: IClickhouseEvent[];+ pending: IClickhouseEvent[];+ } {+ // Ensure events are sorted by created_at+ events.sort(+ (a, b) =>+ new Date(a.created_at || 0).getTime() -+ new Date(b.created_at || 0).getTime(),+ );++ const flush: IClickhouseEvent[] = [];+ const pending: IClickhouseEvent[] = [];++ for (let i = 0; i < events.length; i++) {+ const event = events[i]!;++ if (event.name === 'session_end') {+ flush.push(event);+ } else {+ // For screen_view events, look for next event+ const next = events[i + 1];+ if (next) {+ if (next.name === 'screen_view') {+ event.duration =+ new Date(next.created_at).getTime() -+ new Date(event.created_at).getTime();+ }+ flush.push(event);+ } else {+ pending.push(event);+ }+ }+ }++ return { flush, pending };+ }++ async tryCleanup() {+ try {+ await runEvery({+ interval: 60 * 60 * 24,+ fn: this.cleanup.bind(this),+ key: `${this.name}-cleanup`,+ });+ } catch (error) {+ this.logger.error('Failed to run cleanup', { error });+ }+ }++ /**+ * Cleanup old events from Redis.+ * For each key (no-session and per-session), remove events older than the cutoff date.+ */+ async cleanup() {+ const redis = getRedisCache();+ const cutoffTime = Date.now() - 1000 * 60 * 60 * 24 * this.daysToKeep;++ try {+ const sessionIds = await redis.zrange(this.sessionSortedKey, 0, -1);++ for (const sessionId of sessionIds) {+ const score = await redis.zscore(this.sessionSortedKey, sessionId);++ if (score) {+ const scoreInt = Number.parseInt(score, 10);+ if (scoreInt < cutoffTime) {+ this.logger.warn('Stale session found', {+ sessionId,+ score,+ createdAt: new Date(Number.parseInt(score, 10)),+ eventsCount: await redis.llen(this.getSessionKey(sessionId)),+ });+ }+ }+ }+ } catch (error) {+ this.logger.error('Failed to cleanup stale sessions', { error });+ }+ }++ /**+ * Retrieve the latest screen_view event for a given project/profile.+ */+ public async getLastScreenView({+ projectId,+ profileId,+ }: {+ projectId: string;+ profileId: string;+ }): Promise<IServiceEvent | null> {+ const redis = getRedisCache();+ const eventStr = await redis.get(+ this.getLastEventKey({ projectId, profileId }),+ );+ if (eventStr) {+ const parsed = getSafeJson<IClickhouseEvent>(eventStr);+ if (parsed) {+ return transformEvent(parsed);+ }+ }+ return null;+ }++ private getLastEventKey({+ projectId,+ profileId,+ }: {+ projectId: string;+ profileId: string;+ }) {+ return `session:last_screen_view:${projectId}:${profileId}`;+ }++ private async processPendingSessionsInBatches(+ redis: ReturnType<typeof getRedisCache>,+ pendingUpdates: Array<{+ sessionId: string;+ snapshotCount: number;+ pending: IClickhouseEvent[];+ }>,+ ) {+ for (const batch of this.chunks(+ pendingUpdates,+ this.updatePendingSessionsBatchSize,+ )) {+ const batchArgs: string[] = [];++ for (const { sessionId, snapshotCount, pending } of batch) {+ const sessionKey = this.getSessionKey(sessionId);+ batchArgs.push(+ sessionKey,+ snapshotCount.toString(),+ pending.length.toString(),+ ...pending.map((e) => JSON.stringify(e)),+ );+ }++ await redis.eval(+ this.batchUpdateSessionsScript,+ 0, // no KEYS needed+ ...batchArgs,+ );+ }+ }++ public async getBufferSizeHeavy() {+ const redis = getRedisCache();+ const pipeline = redis.pipeline();++ // Queue up commands in the pipeline+ pipeline.llen(this.regularQueueKey);+ pipeline.zcard(this.sessionSortedKey);++ // Execute pipeline to get initial counts+ const [regularQueueCount, sessionCount] = (await pipeline.exec()) as [+ any,+ any,+ ];++ if (sessionCount[1] === 0) {+ return regularQueueCount[1];+ }++ // Get all session IDs and queue up LLEN commands for each session+ const sessionIds = await redis.zrange(this.sessionSortedKey, 0, -1);+ const sessionPipeline = redis.pipeline();++ for (const sessionId of sessionIds) {+ sessionPipeline.llen(this.getSessionKey(sessionId));+ }++ // Execute all LLEN commands in a single pipeline+ const sessionCounts = (await sessionPipeline.exec()) as [any, any][];++ // Sum up all counts+ const totalSessionEvents = sessionCounts.reduce((sum, [err, count]) => {+ if (err) return sum;+ return sum + count;+ }, 0);++ return regularQueueCount[1] + totalSessionEvents;+ }++ public async getBufferSize() {+ const cached = await getRedisCache().get('event_buffer:cached_count');+ if (cached) {+ return Number.parseInt(cached, 10);+ }+ const count = await this.getBufferSizeHeavy();+ await getRedisCache().set(+ 'event_buffer:cached_count',+ count.toString(),+ 'EX',+ 15, // increase when we know it's stable+ );+ return count;+ }+}diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsdeleted file mode 100644index dcadb3561..000000000--- a/packages/db/src/buffers/event-buffer.ts+++ /dev/null@@ -1,281 +0,0 @@-import { groupBy, omit } from 'ramda';-import SuperJSON from 'superjson';--import { deepMergeObjects, getSafeJson } from '@openpanel/common';-import { getRedisCache, getRedisPub } from '@openpanel/redis';--import {- TABLE_NAMES,- ch,- convertClickhouseDateToJs,-} from '../clickhouse-client';-import { transformEvent } from '../services/event.service';-import type {- IClickhouseEvent,- IServiceEvent,-} from '../services/event.service';-import type { Find, FindMany } from './buffer';-import { RedisBuffer } from './buffer';-import { EventBuffer as NewEventBuffer } from './event-buffer-psql';--const STALLED_QUEUE_TIMEOUT = 1000 * 60 * 60 * 24;--const testNewEventBuffer = new NewEventBuffer();--type BufferType = IClickhouseEvent;-export class EventBuffer extends RedisBuffer<BufferType> {- constructor() {- super('events_v2', null);- }-- getLastEventKey({- projectId,- profileId,- }: {- projectId: string;- profileId: string;- }) {- return `session:last_screen_view:${projectId}:${profileId}`;- }-- public async getLastScreenView({- projectId,- profileId,- }: {- projectId: string;- profileId: string;- }): Promise<IServiceEvent | null> {- const event = await getRedisCache().get(- this.getLastEventKey({ projectId, profileId }),- );-- if (event) {- const parsed = getSafeJson<BufferType>(event);- if (parsed) {- return transformEvent(parsed);- }- }- return null;- }-- public async add(event: BufferType) {- await super.add(event);- if (process.env.TEST_NEW_BUFFER) {- await testNewEventBuffer.add(event);- }- if (event.name === 'screen_view') {- await getRedisCache().set(- this.getLastEventKey({- projectId: event.project_id,- profileId: event.profile_id,- }),- JSON.stringify(event),- 'EX',- 60 * 31,- );- }- }-- public onAdd(event: BufferType) {- getRedisPub().publish(- 'event:received',- SuperJSON.stringify(transformEvent(event)),- );- if (event.profile_id) {- getRedisCache().set(- `live:event:${event.project_id}:${event.profile_id}`,- '',- 'EX',- 60 * 5,- );- }- }-- public onInsert(items: BufferType[]) {- for (const event of items) {- getRedisPub().publish(- 'event:saved',- SuperJSON.stringify(transformEvent(event)),- );- }- }-- protected async processItems(- queue: BufferType[],- ): Promise<{ toInsert: BufferType[]; toKeep: BufferType[] }> {- const toInsert: BufferType[] = [];- const toStalled: BufferType[] = [];-- // Sort data by created_at- // oldest first- queue.sort(sortOldestFirst);-- // All events thats not a screen_view can be sent to clickhouse- // We only need screen_views since we want to calculate the duration of each screen- // To do this we need a minimum of 2 screen_views- queue- .filter((item) => item.name !== 'screen_view' || item.device === 'server')- .forEach((item, index) => {- // Find the last event with data and merge it with the current event- // We use profile_id here since this property can be set from backend as well- const lastEventWithData = queue- .slice(0, index)- .findLast((lastEvent) => {- return (- lastEvent.project_id === item.project_id &&- lastEvent.profile_id === item.profile_id &&- lastEvent.path !== '' &&- lastEvent.name === 'screen_view'- );- });-- const event = deepMergeObjects<BufferType>(- omit(['properties', 'duration'], lastEventWithData || {}),- item,- );-- if (!event.properties) {- event.properties = {};- }-- if (lastEventWithData) {- event.properties.__properties_from = lastEventWithData.id;- }-- return toInsert.push(event);- });-- // Group screen_view events by session_id- const grouped = groupBy(- (item) => item.session_id,- queue.filter(- (item) => item.name === 'screen_view' && item.device !== 'server',- ),- );-- // Iterate over each group- for (const [sessionId, screenViews] of Object.entries(grouped)) {- if (sessionId === '' || !sessionId) {- continue;- }-- // If there is only one screen_view event we can send it back to redis since we can't calculate the duration- const hasSessionEnd = queue.find(- (item) => item.name === 'session_end' && item.session_id === sessionId,- );-- screenViews- ?.slice()- .sort(sortOldestFirst)- .forEach((item, index) => {- const nextScreenView = screenViews[index + 1];- // if nextScreenView does not exists we can't calculate the duration (last event in session)- if (nextScreenView) {- const duration =- new Date(nextScreenView.created_at).getTime() -- new Date(item.created_at).getTime();- const event = {- ...item,- properties: {- ...(item?.properties || {}),- __duration_from: nextScreenView.id,- },- duration,- };- toInsert.push(event);- } else if (hasSessionEnd) {- // push last event in session if we have a session_end event- toInsert.push(item);- }- });- } // for of end-- // Check if we have any events that has been in the queue for more than 24 hour- // This should not theoretically happen but if it does we should move them to stalled- queue.forEach((item) => {- if (- !toInsert.find((i) => i.id === item.id) &&- convertClickhouseDateToJs(item.created_at).getTime() <- new Date().getTime() - STALLED_QUEUE_TIMEOUT- ) {- toStalled.push(item);- }- });-- if (toStalled.length > 0) {- try {- this.logger.info(`Pushing to stalled queue (${toStalled.length})`, {- items: toStalled,- count: toStalled.length,- });- await getRedisCache().rpush(- this.getKey('stalled'),- ...toStalled.map((item) => JSON.stringify(item)),- );- } catch (error) {- toStalled.length = 0;- this.logger.error('Failed to push to stalled queue', { error });- }- }-- return {- toInsert,- toKeep: queue.filter((item) => {- const willBeInserted = toInsert.find((i) => i.id === item.id);- const willBeStalled = toStalled.find((i) => i.id === item.id);- return willBeInserted === undefined && willBeStalled === undefined;- }),- };- }-- private getChunks(items: BufferType[], size: number) {- const chunks = [];- for (let i = 0; i < items.length; i += size) {- chunks.push(items.slice(i, i + size));- }- return chunks;- }-- protected async insertIntoDB(items: BufferType[]): Promise<void> {- for (const chunk of this.getChunks(items, 500)) {- await ch.insert({- table: TABLE_NAMES.events,- values: chunk,- format: 'JSONEachRow',- });- await new Promise((resolve) => setTimeout(resolve, 50));- }- }-- public findMany: FindMany<IClickhouseEvent, IServiceEvent> = async (- callback,- ) => {- if (await this.waitForReleasedLock()) {- return this.getQueue()- .then((queue) => {- return queue.filter(callback).map(transformEvent);- })- .catch(() => {- return [];- });- }- return [];- };-- public find: Find<IClickhouseEvent, IServiceEvent> = async (callback) => {- if (await this.waitForReleasedLock()) {- return this.getQueue(-1)- .then((queue) => {- const match = queue.find(callback);- return match ? transformEvent(match) : null;- })- .catch(() => {- return null;- });- }- return null;- };-}--const sortOldestFirst = (a: IClickhouseEvent, b: IClickhouseEvent) =>- new Date(a.created_at).getTime() - new Date(b.created_at).getTime();diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex ee4924f49..41536cce4 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -1,16 +1,16 @@-import { BotBuffer } from './bot-buffer';-import { BotBuffer as NewBotBuffer } from './bot-buffer-psql';-import { EventBuffer } from './event-buffer';-import { EventBuffer as NewEventBuffer } from './event-buffer-psql';-import { ProfileBuffer } from './profile-buffer';-import { ProfileBuffer as NewProfileBuffer } from './profile-buffer-psql';+import { BotBuffer as BotBufferPsql } from './bot-buffer-psql';+import { BotBuffer as BotBufferRedis } from './bot-buffer-redis';+import { EventBuffer as EventBufferPsql } from './event-buffer-psql';+import { EventBuffer as EventBufferRedis } from './event-buffer-redis';+import { ProfileBuffer as ProfileBufferPsql } from './profile-buffer-psql';+import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer-redis';export const eventBuffer = process.env.USE_NEW_BUFFER- ? new NewEventBuffer()- : new EventBuffer();+ ? new EventBufferRedis()+ : new EventBufferPsql();export const profileBuffer = process.env.USE_NEW_BUFFER- ? new NewProfileBuffer()- : new ProfileBuffer();+ ? new ProfileBufferRedis()+ : new ProfileBufferPsql();export const botBuffer = process.env.USE_NEW_BUFFER- ? new NewBotBuffer()- : new BotBuffer();+ ? new BotBufferRedis()+ : new BotBufferPsql();diff --git a/packages/db/src/buffers/partial-json-match.ts b/packages/db/src/buffers/partial-json-match.tsnew file mode 100644index 000000000..84e0340ac--- /dev/null+++ b/packages/db/src/buffers/partial-json-match.ts@@ -0,0 +1,42 @@+/**+ * Checks if an object partially matches another object, including nested properties+ * @param source The object to check against+ * @param partial The partial object to match+ * @returns boolean indicating if the partial object matches the source+ */+export function isPartialMatch(source: any, partial: any): boolean {+ // Handle null/undefined cases+ if (partial === null || partial === undefined) {+ return source === partial;+ }++ // If partial is not an object, do direct comparison+ if (typeof partial !== 'object') {+ return source === partial;+ }++ // If source is null/undefined but partial is an object, no match+ if (source === null || source === undefined) {+ return false;+ }++ // Check each property in partial+ for (const key in partial) {+ if (+ Object.prototype.hasOwnProperty.call(partial, key) &&+ partial[key] !== undefined+ ) {+ // If property doesn't exist in source, no match+ if (!(key in source)) {+ return false;+ }++ // Recursively check nested objects+ if (!isPartialMatch(source[key], partial[key])) {+ return false;+ }+ }+ }++ return true;+}diff --git a/packages/db/src/buffers/profile-buffer-psql.ts b/packages/db/src/buffers/profile-buffer-psql.tsindex 1ab4c575c..3163bd897 100644--- a/packages/db/src/buffers/profile-buffer-psql.ts+++ b/packages/db/src/buffers/profile-buffer-psql.ts@@ -280,4 +280,12 @@ export class ProfileBuffer extends BaseBuffer {this.logger.info('Cleaned up old profiles', { deleted: deleted.count });}++ async getBufferSize() {+ return db.profileBuffer.count({+ where: {+ processedAt: null,+ },+ });+ }}diff --git a/packages/db/src/buffers/profile-buffer-redis.ts b/packages/db/src/buffers/profile-buffer-redis.tsnew file mode 100644index 000000000..337d78eaf--- /dev/null+++ b/packages/db/src/buffers/profile-buffer-redis.ts@@ -0,0 +1,224 @@+import { createHash } from 'node:crypto';+import { getSafeJson } from '@openpanel/common';+import { type Redis, getRedisCache } from '@openpanel/redis';+import { dissocPath, mergeDeepRight, omit, whereEq } from 'ramda';++import { TABLE_NAMES, ch, chQuery } from '../clickhouse-client';+import type { IClickhouseProfile } from '../services/profile.service';+import { BaseBuffer } from './base-buffer';+import { isPartialMatch } from './partial-json-match';++export class ProfileBuffer extends BaseBuffer {+ private batchSize = process.env.PROFILE_BUFFER_BATCH_SIZE+ ? Number.parseInt(process.env.PROFILE_BUFFER_BATCH_SIZE, 10)+ : 200;+ private daysToKeep = process.env.PROFILE_BUFFER_DAYS_TO_KEEP+ ? Number.parseInt(process.env.PROFILE_BUFFER_DAYS_TO_KEEP, 10)+ : 7;+ private chunkSize = process.env.PROFILE_BUFFER_CHUNK_SIZE+ ? Number.parseInt(process.env.PROFILE_BUFFER_CHUNK_SIZE, 10)+ : 1000;++ private readonly redisBufferKey = 'profile-buffer';+ private readonly redisProfilePrefix = 'profile-cache:';++ private redis: Redis;++ constructor() {+ super({+ name: 'profile',+ onFlush: async () => {+ await this.processBuffer();+ },+ });+ this.redis = getRedisCache();+ }++ private excludeKeys(+ profile: IClickhouseProfile,+ exclude: string[][],+ ): IClickhouseProfile {+ let filtered = profile;+ for (const path of exclude) {+ filtered = dissocPath(path, filtered);+ }+ return filtered;+ }++ private match(source: any, partial: any): boolean {+ const exclude = [+ ['created_at'],+ ['properties', 'browser_version'],+ ['properties', 'browserVersion'],+ ['properties', 'latitude'],+ ['properties', 'longitude'],+ ['properties', 'os_version'],+ ['properties', 'osVersion'],+ ['properties', 'path'],+ ['properties', 'referrer_name'],+ ['properties', 'referrerName'],+ ['properties', 'referrer_type'],+ ['properties', 'referrerType'],+ ['properties', 'referrer'],+ ];++ return isPartialMatch(source, this.excludeKeys(partial, exclude));+ }++ async add(profile: IClickhouseProfile) {+ try {+ this.logger.debug('Adding profile', {+ projectId: profile.project_id,+ profileId: profile.id,+ profile,+ });+ const cacheKey = `${this.redisProfilePrefix}${profile.project_id}:${profile.id}`;++ // Check if we have this profile in Redis cache+ const existingProfile = await this.redis.get(cacheKey);+ let mergedProfile = profile;++ if (!existingProfile) {+ this.logger.debug('Profile not found in cache, checking Clickhouse', {+ projectId: profile.project_id,+ profileId: profile.id,+ });+ // If not in cache, check Clickhouse+ const clickhouseProfile = await this.fetchFromClickhouse(profile);+ if (clickhouseProfile) {+ this.logger.debug('Found existing profile in Clickhouse, merging', {+ projectId: profile.project_id,+ profileId: profile.id,+ });+ mergedProfile = mergeDeepRight(clickhouseProfile, profile);+ }+ } else {+ const parsedProfile = getSafeJson<IClickhouseProfile>(existingProfile);++ if (parsedProfile) {+ // Only merge if checksums are different+ if (this.match(parsedProfile, profile)) {+ return; // Skip if checksums match+ }++ this.logger.debug('Profile changed, merging with cached version', {+ existingProfile: parsedProfile,+ incomingProfile: profile,+ });+ mergedProfile = mergeDeepRight(parsedProfile, profile);+ }+ }++ const result = await this.redis+ .multi()+ .set(+ cacheKey,+ JSON.stringify(mergedProfile),+ 'EX',+ 60 * 60 * 24 * this.daysToKeep,+ )+ .rpush(this.redisBufferKey, JSON.stringify(mergedProfile))+ .llen(this.redisBufferKey)+ .exec();+ if (!result) {+ this.logger.error('Failed to add profile to Redis', {+ profile,+ cacheKey,+ });+ return;+ }+ const bufferLength = (result?.[2]?.[1] as number) ?? 0;++ this.logger.debug('Current buffer length', {+ bufferLength,+ batchSize: this.batchSize,+ });+ if (bufferLength >= this.batchSize) {+ this.logger.info('Buffer full, initiating flush');+ await this.tryFlush();+ }+ } catch (error) {+ this.logger.error('Failed to add profile', { error, profile });+ }+ }++ private async fetchFromClickhouse(+ profile: IClickhouseProfile,+ ): Promise<IClickhouseProfile | null> {+ this.logger.debug('Fetching profile from Clickhouse', {+ projectId: profile.project_id,+ profileId: profile.id,+ });+ const result = await chQuery<IClickhouseProfile>(+ `SELECT *+ FROM ${TABLE_NAMES.profiles}+ WHERE project_id = '${profile.project_id}'+ AND id = '${profile.id}'+ ${+ profile.is_external === false+ ? 'AND created_at > now() - INTERVAL 2 DAY'+ : ''+ }+ ORDER BY created_at DESC+ LIMIT 1`,+ );++ this.logger.debug('Clickhouse fetch result', {+ found: !!result[0],+ projectId: profile.project_id,+ profileId: profile.id,+ });+ return result[0] || null;+ }++ async processBuffer() {+ try {+ this.logger.info('Starting profile buffer processing');+ const profiles = await this.redis.lrange(+ this.redisBufferKey,+ 0,+ this.batchSize - 1,+ );++ if (profiles.length === 0) {+ this.logger.debug('No profiles to process');+ return;+ }++ this.logger.info(`Processing ${profiles.length} profiles in buffer`);+ const parsedProfiles = profiles.map((p) =>+ getSafeJson<IClickhouseProfile>(p),+ );++ let processedChunks = 0;+ for (const chunk of this.chunks(parsedProfiles, this.chunkSize)) {+ processedChunks++;+ this.logger.debug(`Processing chunk ${processedChunks}`, {+ size: chunk.length,+ });+ this.logger.debug('Chunk data', { chunk });++ await ch.insert({+ table: TABLE_NAMES.profiles,+ values: chunk,+ format: 'JSONEachRow',+ });+ this.logger.debug(`Successfully inserted chunk ${processedChunks}`);+ }++ // Only remove profiles after successful insert+ await this.redis.ltrim(this.redisBufferKey, profiles.length, -1);++ this.logger.info('Successfully completed profile processing', {+ totalProfiles: profiles.length,+ totalChunks: processedChunks,+ });+ } catch (error) {+ this.logger.error('Failed to process buffer', { error });+ }+ }++ async getBufferSize() {+ return getRedisCache().llen(this.redisBufferKey);+ }+}diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsdeleted file mode 100644index deb2fe615..000000000--- a/packages/db/src/buffers/profile-buffer.ts+++ /dev/null@@ -1,241 +0,0 @@-import { groupBy, mergeDeepRight, prop } from 'ramda';--import { toDots } from '@openpanel/common';-import { getRedisCache } from '@openpanel/redis';--import { escape } from 'sqlstring';-import {- TABLE_NAMES,- ch,- chQuery,- formatClickhouseDate,-} from '../clickhouse-client';-import { transformProfile } from '../services/profile.service';-import type {- IClickhouseProfile,- IServiceProfile,-} from '../services/profile.service';-import type { Find, FindMany } from './buffer';-import { RedisBuffer } from './buffer';-import { ProfileBuffer as NewProfileBuffer } from './profile-buffer-psql';-const BATCH_SIZE = process.env.BATCH_SIZE_PROFILES- ? Number.parseInt(process.env.BATCH_SIZE_PROFILES, 10)- : 50;--const testNewProfileBuffer = new NewProfileBuffer();--type BufferType = IClickhouseProfile;-export class ProfileBuffer extends RedisBuffer<BufferType> {- constructor() {- super('profiles', BATCH_SIZE);- }-- async add(profile: BufferType) {- await super.add(profile);- if (process.env.TEST_NEW_BUFFER) {- await testNewProfileBuffer.add(profile);- }- }-- // this will do a couple of things:- // - we slice the queue to maxBufferSize since this queries have a limit on character count- // - check redis cache for profiles- // - fetch missing profiles from clickhouse- // - merge the incoming profile with existing data- protected async processItems(- items: BufferType[],- ): Promise<{ toInsert: BufferType[]; toKeep: BufferType[] }> {- const queue = this.combineQueueItems(items);- const slicedQueue = this.maxBufferSize- ? queue.slice(0, this.maxBufferSize)- : queue;- const redisProfiles = await this.getCachedProfiles(slicedQueue);- const dbProfiles = await this.fetchDbProfiles(- slicedQueue.filter((_, index) => !redisProfiles[index]),- );-- const toInsert = this.createProfileValues(- slicedQueue,- redisProfiles,- dbProfiles,- );-- if (toInsert.length > 0) {- await this.updateRedisCache(toInsert);- }-- return Promise.resolve({- toInsert,- toKeep: this.maxBufferSize ? queue.slice(this.maxBufferSize) : [],- });- }-- private combineQueueItems(queue: BufferType[]): BufferType[] {- const itemsToClickhouse = new Map<string, BufferType>();-- queue.forEach((item) => {- const key = item.project_id + item.id;- const existing = itemsToClickhouse.get(key);- itemsToClickhouse.set(key, mergeDeepRight(existing ?? {}, item));- });-- return Array.from(itemsToClickhouse.values());- }-- protected async insertIntoDB(items: BufferType[]): Promise<void> {- await ch.insert({- table: TABLE_NAMES.profiles,- values: items.map((item) => ({- ...item,- created_at: item.created_at- ? formatClickhouseDate(item.created_at)- : '',- })),- format: 'JSONEachRow',- });- }-- private matchPartialObject(- full: any,- partial: any,- options: { ignore: string[] },- ): boolean {- if (typeof partial !== 'object' || partial === null) {- return partial === full;- }-- for (const key in partial) {- if (options.ignore.includes(key)) {- continue;- }-- if (- !(key in full) ||- !this.matchPartialObject(full[key], partial[key], options)- ) {- return false;- }- }-- return true;- }-- private async getCachedProfiles(- queue: BufferType[],- ): Promise<(IClickhouseProfile | null)[]> {- const redisCache = getRedisCache();- const keys = queue.map((item) => `profile:${item.project_id}:${item.id}`);-- if (keys.length === 0) {- return [];- }-- const cachedProfiles = await redisCache.mget(...keys);- return cachedProfiles.map((profile) => {- try {- return profile ? JSON.parse(profile) : null;- } catch (error) {- return null;- }- });- }-- private async fetchDbProfiles(- queue: IClickhouseProfile[],- ): Promise<IClickhouseProfile[]> {- if (queue.length === 0) {- return [];- }-- // const grouped = groupBy(prop('project_id'), queue);- // const queries = Object.entries(grouped).map(([project_id, items]) => {- // if (!items) {- // return [];- // }-- // return chQuery<IClickhouseProfile>(- // `SELECT- // *- // FROM ${TABLE_NAMES.profiles}- // WHERE- // id IN (${items.map((item) => escape(item.id)).join(',')})- // AND created_at > INTERVAL 12 MONTH- // ORDER BY- // created_at DESC`,- // );- // });-- return await chQuery<IClickhouseProfile>(- `SELECT- *- FROM ${TABLE_NAMES.profiles}- WHERE- (project_id, id) IN (${queue.map((item) => `('${item.project_id}', '${item.id}')`).join(',')})- ORDER BY- created_at DESC`,- );- }-- private createProfileValues(- queue: IClickhouseProfile[],- redisProfiles: (IClickhouseProfile | null)[],- dbProfiles: IClickhouseProfile[],- ): IClickhouseProfile[] {- return queue- .map((item, index) => {- const cachedProfile = redisProfiles[index];- const dbProfile = dbProfiles.find(- (p) => p.id === item.id && p.project_id === item.project_id,- );- const profile = cachedProfile || dbProfile;-- if (- profile &&- this.matchPartialObject(- profile,- {- ...item,- properties: toDots(item.properties),- },- {- ignore: ['created_at'],- },- )- ) {- this.logger.debug('No changes for profile', {- profile,- });- return null;- }-- return {- id: item.id,- first_name: item.first_name ?? profile?.first_name ?? '',- last_name: item.last_name ?? profile?.last_name ?? '',- email: item.email ?? profile?.email ?? '',- avatar: item.avatar ?? profile?.avatar ?? '',- properties: toDots({- ...(profile?.properties ?? {}),- ...(item.properties ?? {}),- }),- project_id: item.project_id ?? profile?.project_id ?? '',- created_at: item.created_at ?? profile?.created_at ?? '',- is_external: item.is_external,- };- })- .flatMap((item) => (item ? [item] : []));- }-- private async updateRedisCache(values: IClickhouseProfile[]): Promise<void> {- const redisCache = getRedisCache();- const multi = redisCache.multi();- values.forEach((value) => {- multi.setex(- `profile:${value.project_id}:${value.id}`,- 60 * 30, // 30 minutes- JSON.stringify(value),- );- });- await multi.exec();- }-}diff --git a/packages/db/src/clickhouse-client.ts b/packages/db/src/clickhouse-client.tsindex 3871dbb78..ada760b59 100644--- a/packages/db/src/clickhouse-client.ts+++ b/packages/db/src/clickhouse-client.ts@@ -86,73 +86,55 @@ const cleanQuery = (query?: string) =>? query.replace(/\n/g, '').replace(/\s+/g, ' ').trim(): undefined;-const createChildLogger = (property: string, args?: any[]) => {- if (property === 'insert') {- return logger.child({- property,- table: args?.[0]?.table,- values: (args?.[0]?.values || []).length,- });+async function withRetry<T>(+ operation: () => Promise<T>,+ maxRetries = 3,+ baseDelay = 500,+): Promise<T> {+ let lastError: Error | undefined;++ for (let attempt = 0; attempt < maxRetries; attempt++) {+ try {+ const res = await operation();+ if (attempt > 0) {+ logger.info('Retry operation succeeded', { attempt });+ }+ return res;+ } catch (error: any) {+ lastError = error;++ if (+ error.message.includes('Connect') ||+ error.message.includes('socket hang up') ||+ error.message.includes('Timeout error')+ ) {+ const delay = baseDelay * 2 ** attempt;+ logger.warn(+ `Attempt ${attempt + 1}/${maxRetries} failed, retrying in ${delay}ms`,+ {+ error: error.message,+ },+ );+ await new Promise((resolve) => setTimeout(resolve, delay));+ continue;+ }++ throw error; // Non-retriable error+ }}- return logger.child({- property,- table: args?.[0]?.table,- query: cleanQuery(args?.[0]?.query),- });-};+ throw lastError;+}export const ch = new Proxy(originalCh, {get(target, property, receiver) {- if (property === 'insert' || property === 'query') {- return async (...args: any[]) => {- const childLogger = createChildLogger(property, args);-- if (property === 'insert') {- childLogger.info('insert info');- }-- try {- // First attempt- if (property in target) {- // @ts-expect-error- return await target[property](...args);- }- } catch (error: unknown) {- if (- error instanceof Error &&- (error.message.includes('Connect') ||- error.message.includes('socket hang up') ||- error.message.includes('Timeout error'))- ) {- childLogger.error('First failed attempt', {- error,- });- await new Promise((resolve) => setTimeout(resolve, 500));- try {- // Retry once- childLogger.info(`Retrying ${property}`);- if (property in target) {- // @ts-expect-error- return await target[property](...args);- }- } catch (retryError) {- childLogger.error('Second failed attempt', retryError);- throw retryError; // Rethrow or handle as needed- }- } else {- childLogger.error('Failed without retry', {- ...args[0],- error,- });-- // Handle other errors or rethrow them- throw error;- }- }- };+ const value = Reflect.get(target, property, receiver);++ if (property === 'insert') {+ return (...args: any[]) => withRetry(() => value.apply(target, args));}- return Reflect.get(target, property, receiver);++ return value;},});