OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(queue): stop daily api crashes from unhandled BullMQ 'error' events
Carl-Gerhard Lindesvärde11ab6262117
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex c3d8b2f1e..14fe72d26 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -28,6 +28,23 @@ function pickShard(projectId: string) {export const queueLogger = createLogger({ name: 'queue' });+// BullMQ re-emits ioredis connection errors on every Queue instance; with no+// 'error' listener Node throws them as uncaughtException and kills the+// process (the api died ~daily from idle-socket ECONNRESETs, see+// api-crash-econnreset-plan.md). ioredis reconnects on its own — log and+// continue.+const guardQueue = <+ T extends { on(event: 'error', listener: (error: Error) => void): unknown },+>(+ queue: T,+ name: string+): T => {+ queue.on('error', (error) => {+ queueLogger.error({ err: error, queue: name }, 'queue connection error');+ });+ return queue;+};+export interface EventsQueuePayloadIncomingEvent {type: 'incomingEvent';payload: {@@ -224,22 +241,25 @@ export const getEventsGroupQueueShard = (groupId: string) => {return queue;};-export const sessionsQueue = new Queue<SessionsQueuePayload>(- getQueueName('sessions'),- {+export const sessionsQueue = guardQueue(+ new Queue<SessionsQueuePayload>(getQueueName('sessions'), {connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: true,},- }+ }),+ 'sessions');-export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {- connection: getRedisQueue(),- defaultJobOptions: {- removeOnComplete: 10,- },-});+export const cronQueue = guardQueue(+ new Queue<CronQueuePayload>(getQueueName('cron'), {+ connection: getRedisQueue(),+ defaultJobOptions: {+ removeOnComplete: 10,+ },+ }),+ 'cron'+);export type NotificationQueuePayload = {type: 'sendNotification';@@ -248,14 +268,14 @@ export type NotificationQueuePayload = {};};-export const notificationQueue = new Queue<NotificationQueuePayload>(- getQueueName('notification'),- {+export const notificationQueue = guardQueue(+ new Queue<NotificationQueuePayload>(getQueueName('notification'), {connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 10,},- }+ }),+ 'notification');export type ImportQueuePayload = {@@ -265,15 +285,15 @@ export type ImportQueuePayload = {};};-export const importQueue = new Queue<ImportQueuePayload>(- getQueueName('import'),- {+export const importQueue = guardQueue(+ new Queue<ImportQueuePayload>(getQueueName('import'), {connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 10,removeOnFail: 50,},- }+ }),+ 'import');export type InsightsQueuePayloadProject = {@@ -281,14 +301,14 @@ export type InsightsQueuePayloadProject = {payload: { projectId: string; date: string };};-export const insightsQueue = new Queue<InsightsQueuePayloadProject>(- getQueueName('insights'),- {+export const insightsQueue = guardQueue(+ new Queue<InsightsQueuePayloadProject>(getQueueName('insights'), {connection: getRedisQueue(),defaultJobOptions: {removeOnComplete: 100,},- }+ }),+ 'insights');export type GscQueuePayloadSync = {@@ -301,21 +321,23 @@ export type GscQueuePayloadBackfill = {};export type GscQueuePayload = GscQueuePayloadSync | GscQueuePayloadBackfill;-export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {- connection: getRedisQueue(),- defaultJobOptions: {- removeOnComplete: 50,- removeOnFail: 100,- },-});+export const gscQueue = guardQueue(+ new Queue<GscQueuePayload>(getQueueName('gsc'), {+ connection: getRedisQueue(),+ defaultJobOptions: {+ removeOnComplete: 50,+ removeOnFail: 100,+ },+ }),+ 'gsc'+);export type CohortComputePayload = {cohortId: string;};-export const cohortComputeQueue = new Queue<CohortComputePayload>(- getQueueName('cohortCompute'),- {+export const cohortComputeQueue = guardQueue(+ new Queue<CohortComputePayload>(getQueueName('cohortCompute'), {connection: getRedisQueue(),defaultJobOptions: {attempts: 3,@@ -326,5 +348,6 @@ export const cohortComputeQueue = new Queue<CohortComputePayload>(removeOnComplete: { age: 3600, count: 100 },removeOnFail: { age: 86400, count: 100 },},- },+ }),+ 'cohortCompute');