OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(api): failed jobs on the sessions, cron, notification and insights queues are trimmed
Carl-Gerhard Lindesvärd5ec565a888c2
diff --git a/packages/core/src/jobs.registry.test.ts b/packages/core/src/jobs.registry.test.tsindex a1ddc3bb7..c1fbf33d0 100644--- a/packages/core/src/jobs.registry.test.ts+++ b/packages/core/src/jobs.registry.test.ts@@ -28,13 +28,22 @@ const GOLDEN_SCHEDULER_IDS = [].sort();// Retry and retention values pinned as they are: five queues never retry and-// four have unbounded failed sets.+// every queue bounds its failed set.const PINNED_DEFAULTS = {- sessions: { removeOnComplete: true },- cron: { removeOnComplete: 10 },- notification: { removeOnComplete: 10 },+ sessions: {+ removeOnComplete: true,+ removeOnFail: { age: 86_400, count: 1000 },+ },+ cron: { removeOnComplete: 10, removeOnFail: { age: 86_400, count: 100 } },+ notification: {+ removeOnComplete: 10,+ removeOnFail: { age: 86_400, count: 100 },+ },import: { removeOnComplete: 10, removeOnFail: 50 },- insights: { removeOnComplete: 100 },+ insights: {+ removeOnComplete: 100,+ removeOnFail: { age: 86_400, count: 100 },+ },gsc: { removeOnComplete: 50, removeOnFail: 100 },cohortCompute: {attempts: 3,diff --git a/packages/core/src/jobs.registry.ts b/packages/core/src/jobs.registry.tsindex 70e875d0b..54183d87d 100644--- a/packages/core/src/jobs.registry.ts+++ b/packages/core/src/jobs.registry.ts@@ -48,8 +48,14 @@ const CONCURRENCY = {cohortCompute: 2,} as const;+// Every queue bounds its failed set: a failed job is a Redis hash with no TTL,+// and `age` alone only trims when another job in the queue finishes, so each+// is paired with a count.const sessions = defineQueue('sessions', {- defaults: { removeOnComplete: true },+ defaults: {+ removeOnComplete: true,+ removeOnFail: { age: DAY_IN_SECONDS, count: 1000 },+ },worker: { concurrency: CONCURRENCY.sessions },compat: legacyCompat.sessions,jobs: {@@ -60,7 +66,10 @@ const sessions = defineQueue('sessions', {// One queue, one worker at concurrency 1, every scheduled job in the system.// Pausing it from bull-board halts all buffer flushing.const cron = defineQueue('cron', {- defaults: { removeOnComplete: 10 },+ defaults: {+ removeOnComplete: 10,+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },+ },worker: { concurrency: CONCURRENCY.cron },compat: legacyCompat.cron,jobs: {@@ -80,7 +89,10 @@ const cron = defineQueue('cron', {});const notification = defineQueue('notification', {- defaults: { removeOnComplete: 10 },+ defaults: {+ removeOnComplete: 10,+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },+ },worker: { concurrency: CONCURRENCY.notification },compat: legacyCompat.notification,jobs: {@@ -98,7 +110,10 @@ const importQueue = defineQueue('import', {});const insights = defineQueue('insights', {- defaults: { removeOnComplete: 100 },+ defaults: {+ removeOnComplete: 100,+ removeOnFail: { age: DAY_IN_SECONDS, count: 100 },+ },worker: { concurrency: CONCURRENCY.insights },compat: legacyCompat.insights,jobs: {@@ -119,8 +134,6 @@ const cohortCompute = defineQueue('cohortCompute', {defaults: {attempts: COHORT_ATTEMPTS,backoff: { type: 'exponential', delay: COHORT_BACKOFF_MS },- // `age` alone only trims when another job in this queue finishes, so it is- // paired with a count bound.removeOnComplete: { age: HOUR_IN_SECONDS, count: 100 },removeOnFail: { age: DAY_IN_SECONDS, count: 100 },},diff --git a/packages/core/src/jobs/compat.ts b/packages/core/src/jobs/compat.tsindex ee85adaad..b95c4308b 100644--- a/packages/core/src/jobs/compat.ts+++ b/packages/core/src/jobs/compat.ts@@ -55,8 +55,7 @@ function discriminatedByType(types: readonly string[]): CompatHook {/*** One hook per queue, keyed by registry key. Each is deletable on its own —- * but not until no legacy-shaped job can still be replayed, and four queues- * have no `removeOnFail` at all, so their failed sets never lapse.+ * but not until no legacy-shaped job can still be replayed from a failed set.*/export const legacyCompat = {// Job name `'session'`, payload type `'createSessionEnd'` — the one queue