Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(api): failed jobs on the sessions, cron, notification and insights queues are trimmed

Carl-Gerhard Lindesvärd5ec565a888c2
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.ts
index a1ddc3bb7..c1fbf33d0 100644
--- a/packages/core/src/jobs.registry.test.ts
+++ b/packages/core/src/jobs.registry.test.ts
@@ -28,13 +28,22 @@ const GOLDEN_SCHEDULER_IDS = [
].sort();
// Retry and retention values pinned as they are: five queues never retry and
-// four have unbounded failed sets.
+// every queue bounds its failed set.
const PINNED_DEFAULTS = {
- sessions: { removeOnComplete: true },
- cron: { removeOnComplete: 10 },
- notification: { removeOnComplete: 10 },
+ sessions: {
+ removeOnComplete: true,
+ removeOnFail: { age: 86_400, count: 1000 },
+ },
+ cron: { removeOnComplete: 10, removeOnFail: { age: 86_400, count: 100 } },
+ notification: {
+ removeOnComplete: 10,
+ removeOnFail: { age: 86_400, count: 100 },
+ },
import: { removeOnComplete: 10, removeOnFail: 50 },
- insights: { removeOnComplete: 100 },
+ insights: {
+ removeOnComplete: 100,
+ removeOnFail: { age: 86_400, count: 100 },
+ },
gsc: { removeOnComplete: 50, removeOnFail: 100 },
cohortCompute: {
attempts: 3,
diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.ts
index 70e875d0b..54183d87d 100644
--- a/packages/core/src/jobs.registry.ts
+++ b/packages/core/src/jobs.registry.ts
@@ -48,8 +48,14 @@ const CONCURRENCY = {
cohortCompute: 2,
} as const;
+// Every queue bounds its failed set: a failed job is a Redis hash with no TTL,
+// and `age` alone only trims when another job in the queue finishes, so each
+// is paired with a count.
const sessions = defineQueue('sessions', {
- defaults: { removeOnComplete: true },
+ defaults: {
+ removeOnComplete: true,
+ removeOnFail: { age: DAY_IN_SECONDS, count: 1000 },
+ },
worker: { concurrency: CONCURRENCY.sessions },
compat: legacyCompat.sessions,
jobs: {
@@ -60,7 +66,10 @@ const sessions = defineQueue('sessions', {
// One queue, one worker at concurrency 1, every scheduled job in the system.
// Pausing it from bull-board halts all buffer flushing.
const cron = defineQueue('cron', {
- defaults: { removeOnComplete: 10 },
+ defaults: {
+ removeOnComplete: 10,
+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },
+ },
worker: { concurrency: CONCURRENCY.cron },
compat: legacyCompat.cron,
jobs: {
@@ -80,7 +89,10 @@ const cron = defineQueue('cron', {
});
const notification = defineQueue('notification', {
- defaults: { removeOnComplete: 10 },
+ defaults: {
+ removeOnComplete: 10,
+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },
+ },
worker: { concurrency: CONCURRENCY.notification },
compat: legacyCompat.notification,
jobs: {
@@ -98,7 +110,10 @@ const importQueue = defineQueue('import', {
});
const insights = defineQueue('insights', {
- defaults: { removeOnComplete: 100 },
+ defaults: {
+ removeOnComplete: 100,
+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },
+ },
worker: { concurrency: CONCURRENCY.insights },
compat: legacyCompat.insights,
jobs: {
@@ -119,8 +134,6 @@ const cohortCompute = defineQueue('cohortCompute', {
defaults: {
attempts: COHORT_ATTEMPTS,
backoff: { type: 'exponential', delay: COHORT_BACKOFF_MS },
- // `age` alone only trims when another job in this queue finishes, so it is
- // paired with a count bound.
removeOnComplete: { age: HOUR_IN_SECONDS, count: 100 },
removeOnFail: { age: DAY_IN_SECONDS, count: 100 },
},
diff --git a/packages/core/src/jobs/compat.ts b/packages/core/src/jobs/compat.ts
index ee85adaad..b95c4308b 100644
--- a/packages/core/src/jobs/compat.ts
+++ b/packages/core/src/jobs/compat.ts
@@ -55,8 +55,7 @@ function discriminatedByType(types: readonly string[]): CompatHook {
/**
* One hook per queue, keyed by registry key. Each is deletable on its own —
- * but not until no legacy-shaped job can still be replayed, and four queues
- * have no `removeOnFail` at all, so their failed sets never lapse.
+ * but not until no legacy-shaped job can still be replayed from a failed set.
*/
export const legacyCompat = {
// Job name `'session'`, payload type `'createSessionEnd'` — the one queue