OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
init redis lazy
Carl-Gerhard Lindesvärdf2298a1b054d
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex 3291aa145..aed924bce 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -1,11 +1,10 @@-import { logger } from '@/utils/logger';import { getClientIp, parseIp } from '@/utils/parseIp';import type { FastifyReply, FastifyRequest } from 'fastify';import { generateDeviceId } from '@openpanel/common';import { getSalts } from '@openpanel/db';import { eventsQueue } from '@openpanel/queue';-import { redis } from '@openpanel/redis';+import { getRedisCache } from '@openpanel/redis';import type { PostEventPayload } from '@openpanel/sdk';export async function postEvent(@@ -38,7 +37,7 @@ export async function postEvent(});// this will ensure that we don't have multiple events creating sessions- const locked = await redis.set(+ const locked = await getRedisCache().set(`request:priority:${currentDeviceId}-${previousDeviceId}`,'locked','EX',diff --git a/apps/api/src/controllers/live.controller.ts b/apps/api/src/controllers/live.controller.tsindex ba783fe22..915fe7e9c 100644--- a/apps/api/src/controllers/live.controller.ts+++ b/apps/api/src/controllers/live.controller.ts@@ -1,6 +1,5 @@import { validateClerkJwt } from '@/utils/auth';import type { FastifyReply, FastifyRequest } from 'fastify';-import { escape } from 'sqlstring';import superjson from 'superjson';import type * as WebSocket from 'ws';@@ -13,7 +12,7 @@ import {TABLE_NAMES,transformMinimalEvent,} from '@openpanel/db';-import { redis, redisPub, redisSub } from '@openpanel/redis';+import { getRedisCache, getRedisPub, getRedisSub } from '@openpanel/redis';import { getProjectAccess } from '@openpanel/trpc';export function getLiveEventInfo(key: string) {@@ -36,8 +35,8 @@ export async function testVisitors(return reply.status(404).send('No event found');}event.projectId = req.params.projectId;- redisPub.publish('event:received', superjson.stringify(event));- redis.set(+ getRedisPub().publish('event:received', superjson.stringify(event));+ getRedisCache().set(`live:event:${event.projectId}:${Math.random() * 1000}`,'','EX',@@ -61,7 +60,7 @@ export async function testEvents(if (!event) {return reply.status(404).send('No event found');}- redisPub.publish('event:saved', superjson.stringify(event));+ getRedisPub().publish('event:saved', superjson.stringify(event));reply.status(202).send(event);}@@ -77,8 +76,8 @@ export function wsVisitors() {const { params } = req;- redisSub.subscribe('event:received');- redisSub.psubscribe('__key*:expired');+ getRedisSub().subscribe('event:received');+ getRedisSub().psubscribe('__key*:expired');const message = (channel: string, message: string) => {if (channel === 'event:received') {@@ -99,14 +98,14 @@ export function wsVisitors(}};- redisSub.on('message', message);- redisSub.on('pmessage', pmessage);+ getRedisSub().on('message', message);+ getRedisSub().on('pmessage', pmessage);connection.socket.on('close', () => {- redisSub.unsubscribe('event:saved');- redisSub.punsubscribe('__key*:expired');- redisSub.off('message', message);- redisSub.off('pmessage', pmessage);+ getRedisSub().unsubscribe('event:saved');+ getRedisSub().punsubscribe('__key*:expired');+ getRedisSub().off('message', message);+ getRedisSub().off('pmessage', pmessage);});}@@ -140,7 +139,7 @@ export async function wsProjectEvents(projectId: params.projectId,});- redisSub.subscribe(subscribeToEvent);+ getRedisSub().subscribe(subscribeToEvent);const message = async (channel: string, message: string) => {const event = getSuperJson<IServiceCreateEventPayload>(message);@@ -159,10 +158,10 @@ export async function wsProjectEvents(}};- redisSub.on('message', message as any);+ getRedisSub().on('message', message as any);connection.socket.on('close', () => {- redisSub.unsubscribe(subscribeToEvent);- redisSub.off('message', message as any);+ getRedisSub().unsubscribe(subscribeToEvent);+ getRedisSub().off('message', message as any);});}diff --git a/apps/api/src/controllers/misc.controller.ts b/apps/api/src/controllers/misc.controller.tsindex 41b172ba3..08861501f 100644--- a/apps/api/src/controllers/misc.controller.ts+++ b/apps/api/src/controllers/misc.controller.ts@@ -5,7 +5,7 @@ import icoToPng from 'ico-to-png';import sharp from 'sharp';import { createHash } from '@openpanel/common';-import { redis } from '@openpanel/redis';+import { getRedisCache } from '@openpanel/redis';interface GetFaviconParams {url: string;@@ -51,7 +51,7 @@ export async function getFavicon() {function sendBuffer(buffer: Buffer, cacheKey?: string) {if (cacheKey) {- redis.set(`favicon:${cacheKey}`, buffer.toString('base64'));+ getRedisCache().set(`favicon:${cacheKey}`, buffer.toString('base64'));}reply.type('image/png');return reply.send(buffer);@@ -65,7 +65,7 @@ export async function getFavicon(if (imageExtensions.find((ext) => url.endsWith(ext))) {const cacheKey = createHash(url, 32);- const cache = await redis.get(`favicon:${cacheKey}`);+ const cache = await getRedisCache().get(`favicon:${cacheKey}`);if (cache) {return sendBuffer(Buffer.from(cache, 'base64'));}@@ -76,7 +76,7 @@ export async function getFavicon(}const { hostname } = new URL(url);- const cache = await redis.get(`favicon:${hostname}`);+ const cache = await getRedisCache().get(`favicon:${hostname}`);if (cache) {return sendBuffer(Buffer.from(cache, 'base64'));@@ -104,9 +104,9 @@ export async function clearFavicons(request: FastifyRequest,reply: FastifyReply) {- const keys = await redis.keys('favicon:*');+ const keys = await getRedisCache().keys('favicon:*');for (const key of keys) {- await redis.del(key);+ await getRedisCache().del(key);}return reply.status(404).send('OK');}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 57e596bc4..f7a984263 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -10,7 +10,7 @@ import { round } from '@openpanel/common';import { chQuery, db, TABLE_NAMES } from '@openpanel/db';import type { IServiceClient } from '@openpanel/db';import { eventsQueue } from '@openpanel/queue';-import { redis, redisPub } from '@openpanel/redis';+import { getRedisCache, getRedisPub } from '@openpanel/redis';import type { AppRouter } from '@openpanel/trpc';import { appRouter, createContext } from '@openpanel/trpc';@@ -98,7 +98,7 @@ const startServer = async () => {reply.send({ name: 'openpanel sdk api' });});fastify.get('/healthcheck', async (request, reply) => {- const redisRes = await withTimings(redis.keys('*'));+ const redisRes = await withTimings(getRedisCache().keys('*'));const dbRes = await withTimings(db.project.findFirst());const queueRes = await withTimings(eventsQueue.getCompleted());const chRes = await withTimings(@@ -150,7 +150,7 @@ const startServer = async () => {});// Notify when keys expires- redisPub.config('SET', 'notify-keyspace-events', 'Ex');+ getRedisPub().config('SET', 'notify-keyspace-events', 'Ex');} catch (e) {logger.error(e, 'Failed to start server');}diff --git a/apps/worker/scripts/debug.ts b/apps/worker/scripts/debug.tsindex 41817eef7..604223e01 100644--- a/apps/worker/scripts/debug.ts+++ b/apps/worker/scripts/debug.ts@@ -3,11 +3,11 @@ import { escape } from 'sqlstring';import type { IClickhouseEvent } from '@openpanel/db';import { chQuery, eventBuffer, TABLE_NAMES } from '@openpanel/db';import { sessionsQueue } from '@openpanel/queue/src/queues';-import { redis } from '@openpanel/redis';+import { getRedisCache, getRedisQueue } from '@openpanel/redis';async function debugStalledEvents() {- const keys = await redis.keys('bull:sessions:sessionEnd*');- const delayedZRange = await redis.zrange(+ const keys = await getRedisQueue().keys('bull:sessions:sessionEnd*');+ const delayedZRange = await getRedisQueue().zrange('bull:sessions:delayed',0,-1,@@ -20,10 +20,14 @@ async function debugStalledEvents() {}return acc;},- [] as Record<string, number>+ {} as Record<string, number>+ );+ const opKeys = await getRedisCache().keys('op:*');+ const stalledEvents = await getRedisCache().lrange(+ 'op:buffer:events:stalled',+ 0,+ -1);- const opKeys = await redis.keys('op:*');- const stalledEvents = await redis.lrange('op:buffer:events:stalled', 0, -1);// keys.forEach((key) => {// console.log(key);// });@@ -112,7 +116,14 @@ async function debugStalledEvents() {delayedJobs.sort((a, b) => a.timestamp + a.delay - (b.timestamp + b.delay));let delayedJobsCount = 0;delayedJobs.forEach((job) => {- const date = new Date(delayedValues[job.id]);+ if (!job.id) {+ return;+ }+ const timestamp = delayedValues[job.id];+ if (!timestamp) {+ return;+ }+ const date = new Date(timestamp);// if date is in the past// if (date.getTime() - 1000 * 60 * 5 < Date.now()) {if (date.getTime() < Date.now()) {diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex 19a29deaa..276b9885e 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -5,12 +5,8 @@ import type { WorkerOptions } from 'bullmq';import { Worker } from 'bullmq';import express from 'express';-import {- connection,- cronQueue,- eventsQueue,- sessionsQueue,-} from '@openpanel/queue';+import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';+import { getRedisQueue } from '@openpanel/redis';import { cronJob } from './jobs/cron';import { eventsJob } from './jobs/events';@@ -23,12 +19,7 @@ serverAdapter.setBasePath(process.env.SELF_HOSTED ? '/worker' : '/');const app = express();const workerOptions: WorkerOptions = {- connection: {- ...connection,- enableReadyCheck: false,- maxRetriesPerRequest: null,- enableOfflineQueue: true,- },+ connection: getRedisQueue(),concurrency: parseInt(process.env.CONCURRENCY || '1', 10),};diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex cb2bfd40d..54ee7eed5 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -14,7 +14,7 @@ import type {EventsQueuePayloadCreateSessionEnd,EventsQueuePayloadIncomingEvent,} from '@openpanel/queue';-import { redis } from '@openpanel/redis';+import { getRedisQueue } from '@openpanel/redis';function noDateInFuture(eventDate: Date): Date {if (eventDate > new Date()) {@@ -217,7 +217,9 @@ async function getSessionEnd({currentDeviceId: string;previousDeviceId: string;}) {- const sessionEndKeys = await redis.keys(`*:sessionEnd:${projectId}:*`);+ const sessionEndKeys = await getRedisQueue().keys(+ `*:sessionEnd:${projectId}:*`+ );const sessionEndJobCurrentDeviceId = await findJobByPrefix(sessionsQueue,diff --git a/apps/worker/src/jobs/events.ts b/apps/worker/src/jobs/events.tsindex a94220020..d18922172 100644--- a/apps/worker/src/jobs/events.ts+++ b/apps/worker/src/jobs/events.ts@@ -7,7 +7,6 @@ import type {EventsQueuePayloadCreateSessionEnd,EventsQueuePayloadIncomingEvent,} from '@openpanel/queue';-import { redis } from '@openpanel/redis';import { createSessionEnd } from './events.create-session-end';import { incomingEvent } from './events.incoming-event';diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex d75f271a1..6d0af53a8 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -1,7 +1,7 @@import client from 'prom-client';import { cronQueue, eventsQueue, sessionsQueue } from '@openpanel/queue';-import { redis } from '@openpanel/redis';+import { getRedisCache } from '@openpanel/redis';const Registry = client.Registry;@@ -75,7 +75,7 @@ buffers.forEach((buffer) => {name: `buffer_${buffer}_count`,help: 'Number of users in the users array',async collect() {- const metric = await redis.llen(`op:buffer:${buffer}`);+ const metric = await getRedisCache().llen(`op:buffer:${buffer}`);this.set(metric);},})@@ -86,7 +86,9 @@ buffers.forEach((buffer) => {name: `buffer_${buffer}_stalled_count`,help: 'Number of users in the users array',async collect() {- const metric = await redis.llen(`op:buffer:${buffer}:stalled`);+ const metric = await getRedisCache().llen(+ `op:buffer:${buffer}:stalled`+ );this.set(metric);},})diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/db/src/buffers/event-buffer.tsindex 78ff5caaa..6fbf09955 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/db/src/buffers/event-buffer.ts@@ -2,7 +2,7 @@ import { groupBy } from 'ramda';import SuperJSON from 'superjson';import { deepMergeObjects } from '@openpanel/common';-import { redis, redisPub } from '@openpanel/redis';+import { getRedisCache, getRedisPub } from '@openpanel/redis';import { ch, TABLE_NAMES } from '../clickhouse-client';import { transformEvent } from '../services/event.service';@@ -31,12 +31,12 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {constructor() {super({table: TABLE_NAMES.events,- redis,+ redis: getRedisCache(),});}public onInsert?: OnInsert<IClickhouseEvent> | undefined = (event) => {- redisPub.publish(+ getRedisPub().publish('event:received',SuperJSON.stringify(transformEvent(event)));@@ -54,7 +54,7 @@ export class EventBuffer extends RedisBuffer<IClickhouseEvent> {savedEvents) => {for (const event of savedEvents) {- redisPub.publish(+ getRedisPub().publish('event:saved',SuperJSON.stringify(transformEvent(event)));diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/db/src/buffers/profile-buffer.tsindex 616b79230..a699ca1e0 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/db/src/buffers/profile-buffer.ts@@ -1,7 +1,7 @@import { mergeDeepRight } from 'ramda';import { toDots } from '@openpanel/common';-import { redis } from '@openpanel/redis';+import { getRedisCache } from '@openpanel/redis';import { ch, chQuery } from '../clickhouse-client';import type {@@ -22,7 +22,7 @@ import { RedisBuffer } from './buffer';export class ProfileBuffer extends RedisBuffer<IClickhouseProfile> {constructor() {super({- redis,+ redis: getRedisCache(),table: 'profiles',batchSize: 100,});diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex 3802929bc..19972c4cf 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -3,7 +3,7 @@ import { escape } from 'sqlstring';import { v4 as uuid } from 'uuid';import { toDots } from '@openpanel/common';-import { redis } from '@openpanel/redis';+import { getRedisCache } from '@openpanel/redis';import type { IChartEventFilter } from '@openpanel/validation';import { eventBuffer } from '../buffers';@@ -184,7 +184,7 @@ export function transformMinimalEvent(}export async function getLiveVisitors(projectId: string) {- const keys = await redis.keys(`live:event:${projectId}:*`);+ const keys = await getRedisCache().keys(`live:event:${projectId}:*`);return keys.length;}diff --git a/packages/queue/index.ts b/packages/queue/index.tsindex 7dba30a62..647ff89f9 100644--- a/packages/queue/index.ts+++ b/packages/queue/index.ts@@ -1,5 +1,4 @@export { eventsQueue, cronQueue, sessionsQueue } from './src/queues';export type * from './src/queues';-export { connection } from './src/connection';export { findJobByPrefix } from './src/utils';export type { JobsOptions } from 'bullmq';diff --git a/packages/queue/package.json b/packages/queue/package.jsonindex ec7f19075..7579e8f6e 100644--- a/packages/queue/package.json+++ b/packages/queue/package.json@@ -9,6 +9,7 @@},"dependencies": {"@openpanel/db": "workspace:*",+ "@openpanel/redis": "workspace:*","bullmq": "^5.8.7"},"devDependencies": {diff --git a/packages/queue/src/connection.ts b/packages/queue/src/connection.tsdeleted file mode 100644index 60a50b38a..000000000--- a/packages/queue/src/connection.ts+++ /dev/null@@ -1,10 +0,0 @@-const parse = (connectionString: string) => {- const url = new URL(connectionString);- return {- host: url.hostname,- port: Number(url.port),- password: url.password,- } as const;-};--export const connection = parse(String(process.env.REDIS_URL));diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 1abd55d2d..f69782884 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -1,10 +1,9 @@import { Queue } from 'bullmq';import type { IServiceCreateEventPayload } from '@openpanel/db';+import { getRedisQueue } from '@openpanel/redis';import type { PostEventPayload } from '@openpanel/sdk';-import { connection } from './connection';-export interface EventsQueuePayloadIncomingEvent {type: 'incomingEvent';payload: {@@ -63,21 +62,21 @@ export type CronQueuePayload =| CronQueuePayloadFlushProfiles;export const eventsQueue = new Queue<EventsQueuePayload>('events', {- connection,+ connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 10,},});export const sessionsQueue = new Queue<SessionsQueuePayload>('sessions', {- connection,+ connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 10,},});export const cronQueue = new Queue<CronQueuePayload>('cron', {- connection,+ connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 10,},diff --git a/packages/redis/cachable.ts b/packages/redis/cachable.tsindex 39fe830d8..4aa663dd9 100644--- a/packages/redis/cachable.ts+++ b/packages/redis/cachable.ts@@ -1,4 +1,4 @@-import { redis } from './redis';+import { getRedisCache } from './redis';export function cacheable<T extends (...args: any) => any>(fn: T,@@ -9,7 +9,7 @@ export function cacheable<T extends (...args: any) => any>(): Promise<Awaited<ReturnType<T>>> {// JSON.stringify here is not bullet proof since ordering of object keys matters etcconst key = `cachable:${fn.name}:${JSON.stringify(args)}`;- const cached = await redis.get(key);+ const cached = await getRedisCache().get(key);if (cached) {try {return JSON.parse(cached);@@ -20,7 +20,7 @@ export function cacheable<T extends (...args: any) => any>(const result = await fn(...(args as any));if (result !== undefined || result !== null) {- redis.setex(key, expire, JSON.stringify(result));+ getRedisCache().setex(key, expire, JSON.stringify(result));}return result;diff --git a/packages/redis/redis.ts b/packages/redis/redis.tsindex cf3e59ecc..7abc06dac 100644--- a/packages/redis/redis.ts+++ b/packages/redis/redis.ts@@ -2,12 +2,65 @@ import type { RedisOptions } from 'ioredis';import Redis from 'ioredis';const options: RedisOptions = {- connectTimeout: 30000,- maxRetriesPerRequest: null,+ connectTimeout: 10000,};export { Redis };-export const redis = new Redis(process.env.REDIS_URL!, options);-export const redisSub = new Redis(process.env.REDIS_URL!, options);-export const redisPub = new Redis(process.env.REDIS_URL!, options);+const createRedisClient = (+ url: string,+ overrides: RedisOptions = {}+): Redis => {+ const client = new Redis(url, { ...options, ...overrides });++ client.on('error', (error) => {+ console.error('Redis Client Error:', error);+ });++ return client;+};++let redisCache: Redis;+export function getRedisCache() {+ if (!redisCache) {+ redisCache = createRedisClient(process.env.REDIS_URL!, options);+ }++ return redisCache;+}++let redisSub: Redis;+export function getRedisSub() {+ if (!redisSub) {+ redisSub = createRedisClient(process.env.REDIS_URL!, options);+ }++ return redisSub;+}++let redisPub: Redis;+export function getRedisPub() {+ if (!redisPub) {+ redisPub = createRedisClient(process.env.REDIS_URL!, options);+ }++ return redisPub;+}++let redisQueue: Redis;+export function getRedisQueue() {+ if (!redisQueue) {+ // Use different redis for queues (self-hosting will re-use the same redis instance)+ redisQueue = createRedisClient(+ (process.env.QUEUE_REDIS_URL || process.env.REDIS_URL)!,+ {+ ...options,+ enableReadyCheck: false,+ maxRetriesPerRequest: null,+ enableOfflineQueue: true,+ }+ );+ }++ return redisQueue;+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex b1c451f13..bab16c73a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1008,6 +1008,9 @@ importers:'@openpanel/db':specifier: workspace:*version: link:../db+ '@openpanel/redis':+ specifier: workspace:*+ version: link:../redisbullmq:specifier: ^5.8.7version: 5.8.7