Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(queue): stop daily api crashes from unhandled BullMQ 'error' events

Carl-Gerhard Lindesvärde11ab6262117
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index c3d8b2f1e..14fe72d26 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -28,6 +28,23 @@ function pickShard(projectId: string) {
export const queueLogger = createLogger({ name: 'queue' });
+// BullMQ re-emits ioredis connection errors on every Queue instance; with no
+// 'error' listener Node throws them as uncaughtException and kills the
+// process (the api died ~daily from idle-socket ECONNRESETs, see
+// api-crash-econnreset-plan.md). ioredis reconnects on its own — log and
+// continue.
+const guardQueue = <
+ T extends { on(event: 'error', listener: (error: Error) => void): unknown },
+>(
+ queue: T,
+ name: string
+): T => {
+ queue.on('error', (error) => {
+ queueLogger.error({ err: error, queue: name }, 'queue connection error');
+ });
+ return queue;
+};
+
export interface EventsQueuePayloadIncomingEvent {
type: 'incomingEvent';
payload: {
@@ -224,22 +241,25 @@ export const getEventsGroupQueueShard = (groupId: string) => {
return queue;
};
-export const sessionsQueue = new Queue<SessionsQueuePayload>(
- getQueueName('sessions'),
- {
+export const sessionsQueue = guardQueue(
+ new Queue<SessionsQueuePayload>(getQueueName('sessions'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: true,
},
- }
+ }),
+ 'sessions'
);
-export const cronQueue = new Queue<CronQueuePayload>(getQueueName('cron'), {
- connection: getRedisQueue(),
- defaultJobOptions: {
- removeOnComplete: 10,
- },
-});
+export const cronQueue = guardQueue(
+ new Queue<CronQueuePayload>(getQueueName('cron'), {
+ connection: getRedisQueue(),
+ defaultJobOptions: {
+ removeOnComplete: 10,
+ },
+ }),
+ 'cron'
+);
export type NotificationQueuePayload = {
type: 'sendNotification';
@@ -248,14 +268,14 @@ export type NotificationQueuePayload = {
};
};
-export const notificationQueue = new Queue<NotificationQueuePayload>(
- getQueueName('notification'),
- {
+export const notificationQueue = guardQueue(
+ new Queue<NotificationQueuePayload>(getQueueName('notification'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 10,
},
- }
+ }),
+ 'notification'
);
export type ImportQueuePayload = {
@@ -265,15 +285,15 @@ export type ImportQueuePayload = {
};
};
-export const importQueue = new Queue<ImportQueuePayload>(
- getQueueName('import'),
- {
+export const importQueue = guardQueue(
+ new Queue<ImportQueuePayload>(getQueueName('import'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 10,
removeOnFail: 50,
},
- }
+ }),
+ 'import'
);
export type InsightsQueuePayloadProject = {
@@ -281,14 +301,14 @@ export type InsightsQueuePayloadProject = {
payload: { projectId: string; date: string };
};
-export const insightsQueue = new Queue<InsightsQueuePayloadProject>(
- getQueueName('insights'),
- {
+export const insightsQueue = guardQueue(
+ new Queue<InsightsQueuePayloadProject>(getQueueName('insights'), {
connection: getRedisQueue(),
defaultJobOptions: {
removeOnComplete: 100,
},
- }
+ }),
+ 'insights'
);
export type GscQueuePayloadSync = {
@@ -301,21 +321,23 @@ export type GscQueuePayloadBackfill = {
};
export type GscQueuePayload = GscQueuePayloadSync | GscQueuePayloadBackfill;
-export const gscQueue = new Queue<GscQueuePayload>(getQueueName('gsc'), {
- connection: getRedisQueue(),
- defaultJobOptions: {
- removeOnComplete: 50,
- removeOnFail: 100,
- },
-});
+export const gscQueue = guardQueue(
+ new Queue<GscQueuePayload>(getQueueName('gsc'), {
+ connection: getRedisQueue(),
+ defaultJobOptions: {
+ removeOnComplete: 50,
+ removeOnFail: 100,
+ },
+ }),
+ 'gsc'
+);
export type CohortComputePayload = {
cohortId: string;
};
-export const cohortComputeQueue = new Queue<CohortComputePayload>(
- getQueueName('cohortCompute'),
- {
+export const cohortComputeQueue = guardQueue(
+ new Queue<CohortComputePayload>(getQueueName('cohortCompute'), {
connection: getRedisQueue(),
defaultJobOptions: {
attempts: 3,
@@ -326,5 +348,6 @@ export const cohortComputeQueue = new Queue<CohortComputePayload>(
removeOnComplete: { age: 3600, count: 100 },
removeOnFail: { age: 86400, count: 100 },
},
- },
+ }),
+ 'cohortCompute'
);