OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(cohorts): stop non-static cohorts from freezing after their first compute (#432)
Carl-Gerhard Lindesvärd6b39c43f237f
diff --git a/apps/worker/src/jobs/cron.cohort-refresh.ts b/apps/worker/src/jobs/cron.cohort-refresh.tsindex 3c9736acf..192522646 100644--- a/apps/worker/src/jobs/cron.cohort-refresh.ts+++ b/apps/worker/src/jobs/cron.cohort-refresh.ts@@ -1,5 +1,4 @@-import { db } from '@openpanel/db';-import { cohortComputeQueue } from '@openpanel/queue';+import { db, enqueueCohortCompute } from '@openpanel/db';export async function cohortRefreshCronJob() {const cohorts = await db.cohort.findMany({@@ -7,17 +6,5 @@ export async function cohortRefreshCronJob() {select: { id: true },});- await Promise.all(- cohorts.map((cohort) =>- cohortComputeQueue.add(- 'cohortCompute',- { cohortId: cohort.id },- {- jobId: `cohort-${cohort.id}`,- removeOnComplete: { age: 3600 },- removeOnFail: { age: 86400 },- },- ),- ),- );+ await Promise.all(cohorts.map((cohort) => enqueueCohortCompute(cohort.id)));}diff --git a/packages/db/src/services/cohort.service.ts b/packages/db/src/services/cohort.service.tsindex 5666ce22b..0e0beafac 100644--- a/packages/db/src/services/cohort.service.ts+++ b/packages/db/src/services/cohort.service.ts@@ -599,24 +599,28 @@ export async function getProfilesInCohort(return new Set(profileIds);}+/**+ * Enqueue a recompute for a cohort.+ *+ * Uses `deduplication` rather than a fixed `jobId`. A fixed jobId makes BullMQ+ * short-circuit `add` for as long as *any* record for that id exists in Redis —+ * and `removeOnComplete: { age }` is not a TTL, it only trims on some other+ * job in the queue finishing. That deadlocks: nothing can be added because the+ * completed record is still there, and the record is never collected because+ * nothing gets added. The deduplication key, in contrast, is released by+ * `moveToFinished` on both completion and terminal failure, so it only collapses+ * a compute that is genuinely still in flight.+ */export async function enqueueCohortCompute(cohortId: string): Promise<void> {await cohortComputeQueue.add('cohortCompute',{ cohortId },{- jobId: `cohort-${cohortId}`,- removeOnComplete: { age: 3600 },- removeOnFail: { age: 86400 },+ deduplication: { id: `cohort-${cohortId}` },},);}-export async function removeCohortComputeJob(cohortId: string): Promise<void> {- await cohortComputeQueue.remove(- `cohort-${cohortId}`,- );-}-export async function listCohortMemberProfiles({projectId,cohortId,diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 9683374ac..c3d8b2f1e 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -320,8 +320,11 @@ export const cohortComputeQueue = new Queue<CohortComputePayload>(defaultJobOptions: {attempts: 3,backoff: { type: 'exponential', delay: 5000 },- removeOnComplete: { age: 3600 },- removeOnFail: { age: 86400 },+ // `age` alone only trims when another job in this queue finishes, so pair+ // it with a count bound to keep the completed/failed sets from growing+ // unbounded during quiet periods.+ removeOnComplete: { age: 3600, count: 100 },+ removeOnFail: { age: 86400, count: 100 },},},);diff --git a/packages/trpc/src/routers/cohort.ts b/packages/trpc/src/routers/cohort.tsindex f8bf73d0b..22ffa06ac 100644--- a/packages/trpc/src/routers/cohort.ts+++ b/packages/trpc/src/routers/cohort.ts@@ -10,7 +10,6 @@ import {getCohortMemberRoutes,getCohortMembers,listCohortMemberProfiles,- removeCohortComputeJob,} from '@openpanel/db';import {type CohortDefinition,@@ -119,11 +118,7 @@ export const cohortRouter = createTRPCRouter({},});- console.log('cohort', cohort);-if (data.definition) {- console.log('enqueueing cohort compute');- await removeCohortComputeJob(cohort.id);await enqueueCohortCompute(cohort.id);}@@ -300,7 +295,6 @@ export const cohortRouter = createTRPCRouter({);}- await removeCohortComputeJob(input.cohortId);await enqueueCohortCompute(input.cohortId);return { success: true };