OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P2-005): jobs/schedulers.ts: the boot-cron port, all 20 ids byte-identical
ralphb92f6e1cb3f8
diff --git a/packages/core/src/jobs/schedulers.test.ts b/packages/core/src/jobs/schedulers.test.tsnew file mode 100644index 000000000..c276c2922--- /dev/null+++ b/packages/core/src/jobs/schedulers.test.ts@@ -0,0 +1,293 @@+import { expect, test } from 'bun:test';+import type { Logger } from '../logger';+import {+ CRON_SCHEDULES,+ PING_SCHEDULE,+ type SchedulerQueue,+ startSchedulers,+} from './schedulers';++// Byte-identity snapshot, mirroring+// `verification/golden/queue-keys/scheduler-ids.json` — which the+// controller's `check.sh` independently recomputes from V1's+// `packages/queue/src/queues.ts` (CronQueuePayload union) and+// `apps/worker/src/boot-cron.ts`. Both sides pin the same 20 ids, including+// the three ADR-005 fixed omissions (ADR-005 acceptance note).+const GOLDEN_SCHEDULER_IDS = [+ 'cohortRefresh',+ 'dataHealth',+ 'delete',+ 'flushEvents',+ 'flushExports',+ 'flushGroups',+ 'flushProfileBackfill',+ 'flushProfiles',+ 'flushReplay',+ 'flushSessions',+ 'gscSync',+ 'insightCleanup',+ 'insightsDaily',+ 'onboarding',+ 'ping',+ 'salt',+ 'sessionReaper',+ 'sessionVacuum',+ 'weeklyDigest',+ 'windDown',+].sort();++function stubLogger(): Logger {+ const noop = () => undefined;+ const logger: Logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+}++/** In-memory stand-in for a BullMQ queue's scheduler surface. */+function fakeQueue(options?: {+ existingSchedulerKeys?: string[];+ conflictOnce?: Set<string>;+ conflictingJobs?: { id: string }[];+}): SchedulerQueue & {+ calls: {+ upserted: { id: string; data: unknown }[];+ removedSchedulers: string[];+ removedJobs: string[];+ };+} {+ const schedulerKeys = new Set(options?.existingSchedulerKeys ?? []);+ const conflictOnce = new Set(options?.conflictOnce ?? []);+ const remainingConflictingJobs = [...(options?.conflictingJobs ?? [])];+ const calls = {+ upserted: [] as { id: string; data: unknown }[],+ removedSchedulers: [] as string[],+ removedJobs: [] as string[],+ };++ return {+ calls,+ async getJobSchedulers() {+ return [...schedulerKeys].map((key) => ({ key }));+ },+ async upsertJobScheduler(id, _repeatOpts, jobTemplate) {+ if (conflictOnce.has(id)) {+ conflictOnce.delete(id);+ throw new Error('Job scheduler job ID already exists');+ }+ schedulerKeys.add(id);+ calls.upserted.push({ id, data: jobTemplate?.data });+ return undefined;+ },+ async removeJobScheduler(id) {+ calls.removedSchedulers.push(id);+ return schedulerKeys.delete(id);+ },+ async getJobs(_types) {+ return remainingConflictingJobs.splice(0).map((job) => ({+ id: job.id,+ remove: async () => {+ calls.removedJobs.push(job.id);+ },+ }));+ },+ };+}++test('the golden scheduler ids are reproduced byte-for-byte', () => {+ const allIds = [...CRON_SCHEDULES.map((s) => s.id), PING_SCHEDULE.id].sort();++ expect(allIds).toHaveLength(20);+ expect(allIds).toEqual(GOLDEN_SCHEDULER_IDS);+});++test('the three ADR-005 fixed omissions are present', () => {+ const ids = new Set(CRON_SCHEDULES.map((s) => s.id));+ expect(ids.has('dataHealth')).toBe(true);+ expect(ids.has('windDown')).toBe(true);+ expect(ids.has('flushExports')).toBe(true);+});++test('ping is excluded by default — neither self-hosted nor production', async () => {+ const queue = fakeQueue();++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ });++ const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();+ expect(upsertedIds).toHaveLength(19);+ expect(upsertedIds).not.toContain('ping');+});++test('ping is excluded when self-hosted but not production', async () => {+ const queue = fakeQueue();++ await startSchedulers({+ queue,+ flags: { selfHosted: true, production: false },+ logger: stubLogger(),+ });++ expect(queue.calls.upserted.map((u) => u.id)).not.toContain('ping');+});++test('ping is excluded when production but not self-hosted', async () => {+ const queue = fakeQueue();++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: true },+ logger: stubLogger(),+ });++ expect(queue.calls.upserted.map((u) => u.id)).not.toContain('ping');+});++test('ping is included only when self-hosted AND production', async () => {+ const queue = fakeQueue();++ await startSchedulers({+ queue,+ flags: { selfHosted: true, production: true },+ logger: stubLogger(),+ });++ const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();+ expect(upsertedIds).toHaveLength(20);+ expect(upsertedIds).toEqual(GOLDEN_SCHEDULER_IDS);+});++test('every upsert carries envelope-shaped data, not a legacy {type} shape', async () => {+ const queue = fakeQueue();++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],+ });++ expect(queue.calls.upserted).toEqual([+ { id: 'salt', data: { payload: null, meta: {} } },+ ]);+});++test('a scheduler no longer declared is pruned', async () => {+ const queue = fakeQueue({+ existingSchedulerKeys: ['salt', 'aRemovedJob'],+ });++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],+ });++ expect(queue.calls.removedSchedulers).toEqual(['aRemovedJob']);+});++test('a declared scheduler that already exists is left alone by pruning', async () => {+ const queue = fakeQueue({ existingSchedulerKeys: ['salt', 'delete'] });++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },+ { id: 'delete', schedule: { pattern: '0 * * * *' } },+ ],+ });++ expect(queue.calls.removedSchedulers).toEqual([]);+});++test('a conflicting upsert is cleaned up and retried, not left failed', async () => {+ const queue = fakeQueue({+ conflictOnce: new Set(['salt']),+ conflictingJobs: [{ id: 'repeat:salt:1700000000000' }],+ });++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],+ });++ // Cleanup removed the stale scheduler-created job and the scheduler+ // itself, then the retry succeeded.+ expect(queue.calls.removedJobs).toEqual(['repeat:salt:1700000000000']);+ expect(queue.calls.removedSchedulers).toContain('salt');+ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['salt']);+});++test('a conflicting job belonging to a different scheduler is left alone', async () => {+ const queue = fakeQueue({+ conflictOnce: new Set(['salt']),+ conflictingJobs: [{ id: 'repeat:delete:1700000000000' }],+ });++ await startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],+ });++ expect(queue.calls.removedJobs).toEqual([]);+ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['salt']);+});++test('a non-conflict upsert error is logged, not thrown — one bad scheduler must not block the rest', async () => {+ const queue = fakeQueue();+ const originalUpsert = queue.upsertJobScheduler.bind(queue);+ queue.upsertJobScheduler = async (id, repeatOpts, jobTemplate) => {+ if (id === 'salt') {+ throw new Error('ECONNRESET');+ }+ return originalUpsert(id, repeatOpts, jobTemplate);+ };++ await expect(+ startSchedulers({+ queue,+ flags: { selfHosted: false, production: false },+ logger: stubLogger(),+ schedulers: [+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },+ { id: 'delete', schedule: { pattern: '0 * * * *' } },+ ],+ })+ ).resolves.toBeUndefined();++ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['delete']);+});++test('idempotent across replicas — a second boot converges on the same set, no duplicates', async () => {+ const queue = fakeQueue();+ const schedulers = [+ { id: 'salt', schedule: { pattern: '0 0 * * *' } as const },+ { id: 'delete', schedule: { pattern: '0 * * * *' } as const },+ ];+ const flags = { selfHosted: false, production: false };++ await startSchedulers({ queue, flags, logger: stubLogger(), schedulers });+ await startSchedulers({ queue, flags, logger: stubLogger(), schedulers });++ const finalKeys = (await queue.getJobSchedulers()).map((s) => s.key).sort();+ expect(finalKeys).toEqual(['delete', 'salt']);+ // Neither run finds a scheduler outside its declared set, so pruning never+ // fires — confirms a second replica's boot neither drops nor duplicates.+ expect(queue.calls.removedSchedulers).toEqual([]);+});diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsnew file mode 100644index 000000000..f6894ad83--- /dev/null+++ b/packages/core/src/jobs/schedulers.ts@@ -0,0 +1,228 @@+// Ports apps/worker/src/boot-cron.ts onto BullMQ's job-scheduler API+// (ADR-005). Upsert is keyed on job name — the V1 scheduler id — which is+// what makes it idempotent across replicas: two processes upserting the same+// id with the same repeat options at boot converge on one scheduler, with no+// locking of our own needed.++import type { Logger } from '../logger';+import { wrap } from './envelope';++export type RepeatSchedule = { pattern: string } | { every: number };++export interface SchedulerDefinition {+ /** V1 scheduler id: the job-scheduler key, and — via BullMQ's own default+ * (`jobName ?? jobSchedulerId`) — the job name each run dispatches on. */+ id: string;+ schedule: RepeatSchedule;+}++export interface SchedulerFlags {+ selfHosted: boolean;+ production: boolean;+}++/**+ * The slice of BullMQ's `Queue` this needs, narrowed so a test can fake it+ * with no Redis connection — mirrors `RunnableJob` in `workers.ts`.+ */+export interface SchedulerQueue {+ getJobSchedulers(): Promise<{ key: string }[]>;+ upsertJobScheduler(+ id: string,+ repeatOpts: RepeatSchedule,+ jobTemplate?: { data?: unknown }+ ): Promise<unknown>;+ removeJobScheduler(id: string): Promise<boolean>;+ getJobs(+ types: readonly ('delayed' | 'waiting' | 'completed' | 'failed')[]+ ): Promise<{ id?: string | null; remove(): Promise<void> }[]>;+}++const CONFLICT_JOB_STATES = [+ 'delayed',+ 'waiting',+ 'completed',+ 'failed',+] as const;++const CONFLICT_ERROR_SUBSTRING = 'job ID already exists';++const MINUTE_MS = 60_000;++// V1's exact 19 always-on scheduler ids and cadences+// (apps/worker/src/boot-cron.ts). `ping` is the 20th and is conditional —+// see `PING_SCHEDULE` and `startSchedulers`.+export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },+ { id: 'delete', schedule: { pattern: '0 * * * *' } },+ { id: 'flushEvents', schedule: { every: 10_000 } },+ { id: 'flushProfiles', schedule: { every: 10_000 } },+ { id: 'flushSessions', schedule: { every: 10_000 } },+ { id: 'flushProfileBackfill', schedule: { every: 30_000 } },+ { id: 'flushReplay', schedule: { every: 10_000 } },+ { id: 'flushGroups', schedule: { every: 10_000 } },+ { id: 'insightsDaily', schedule: { pattern: '0 2 * * *' } },+ { id: 'onboarding', schedule: { pattern: '0 * * * *' } },+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },+ { id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },+ { id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },+ // Daily 04:00 UTC — backstop for cleanup leaks.+ { id: 'sessionVacuum', schedule: { pattern: '0 4 * * *' } },+ // Daily 04:30 UTC — prunes stale insights/events.+ { id: 'insightCleanup', schedule: { pattern: '30 4 * * *' } },+ // Mondays 08:00 UTC — weekly analytics digest email.+ { id: 'weeklyDigest', schedule: { pattern: '0 8 * * 1' } },+ // Daily 07:30 UTC — no-data / data-stopped rescue emails. Owner: misc+ // (ADR-005 acceptance note completes the ownership map).+ { id: 'dataHealth', schedule: { pattern: '30 7 * * *' } },+ // Hourly — expired-trial wind-down emails, block, delete. Owner: organization.+ { id: 'windDown', schedule: { pattern: '0 * * * *' } },+ // Every 1 minute — drains export buffers to S3/GCS. Owner: integration.+ { id: 'flushExports', schedule: { every: MINUTE_MS } },+];++// V1 gated this on `SELF_HOSTED && NODE_ENV === 'production'`+// (apps/worker/src/boot-cron.ts:133).+export const PING_SCHEDULE: SchedulerDefinition = {+ id: 'ping',+ schedule: { pattern: '0 0 * * *' },+};++export interface StartSchedulersOptions {+ queue: SchedulerQueue;+ flags: SchedulerFlags;+ logger: Logger;+ /** Defaults to the V1 cron set; overridable so a test can use a small fixture. */+ schedulers?: readonly SchedulerDefinition[];+}++/**+ * Upserts every declared scheduler, keyed on job name, and prunes any+ * scheduler in Redis that is no longer declared — so an owner removed from+ * `schedulers` stops running rather than lingering forever.+ *+ * Safe to call from every replica at boot: `upsertJobScheduler` is BullMQ's+ * own idempotent operation, and pruning only ever removes schedulers this+ * call does not want, never ones another replica is mid-upserting.+ */+export async function startSchedulers({+ queue,+ flags,+ logger,+ schedulers = CRON_SCHEDULES,+}: StartSchedulersOptions): Promise<void> {+ const desired =+ flags.selfHosted && flags.production+ ? [...schedulers, PING_SCHEDULE]+ : schedulers;++ logger.info('updating cron jobs');++ await pruneStaleSchedulers(queue, new Set(desired.map((s) => s.id)), logger);++ for (const scheduler of desired) {+ await upsertWithConflictRetry(queue, scheduler, logger);+ }+}++async function pruneStaleSchedulers(+ queue: SchedulerQueue,+ desiredIds: Set<string>,+ logger: Logger+): Promise<void> {+ const current = await queue.getJobSchedulers().catch((error) => {+ logger.error({ err: error }, 'error getting job schedulers');+ return [];+ });++ for (const scheduler of current) {+ if (desiredIds.has(scheduler.key)) {+ continue;+ }+ await queue.removeJobScheduler(scheduler.key).catch((error) => {+ logger.error(+ { err: error, jobScheduler: scheduler.key },+ 'error removing job scheduler'+ );+ });+ }+}++async function upsertWithConflictRetry(+ queue: SchedulerQueue,+ scheduler: SchedulerDefinition,+ logger: Logger+): Promise<void> {+ try {+ await upsert(queue, scheduler);+ } catch (error) {+ if (!isConflictError(error)) {+ logger.error(+ { err: error, job: scheduler.id },+ 'error upserting job scheduler'+ );+ return;+ }++ logger.warn(+ { job: scheduler.id },+ 'job scheduler conflict detected, attempting cleanup'+ );+ await removeConflictingJobs(queue, scheduler.id, logger);+ // Also try removing the scheduler itself to start fresh.+ await queue.removeJobScheduler(scheduler.id).catch(() => undefined);++ try {+ await upsert(queue, scheduler);+ logger.info({ job: scheduler.id }, 'job scheduler created after cleanup');+ } catch (retryError) {+ logger.error(+ { err: retryError, job: scheduler.id },+ 'error upserting job scheduler after cleanup'+ );+ }+ }+}++function upsert(+ queue: SchedulerQueue,+ scheduler: SchedulerDefinition+): Promise<unknown> {+ // Envelope-shaped for every scheduler-created job — these are freshly+ // enqueued, never a replay of a V1-shaped job, so there is nothing for the+ // `cron` compat hook to do here (ADR-005: new data only, old shape read+ // back through `resolveJob`).+ return queue.upsertJobScheduler(scheduler.id, scheduler.schedule, {+ data: wrap(null, {}),+ });+}++function isConflictError(error: unknown): boolean {+ return (+ error instanceof Error && error.message.includes(CONFLICT_ERROR_SUBSTRING)+ );+}++/** BullMQ scheduler-created job ids look like `repeat:<key>:<timestamp>`. */+async function removeConflictingJobs(+ queue: SchedulerQueue,+ schedulerKey: string,+ logger: Logger+): Promise<void> {+ for (const state of CONFLICT_JOB_STATES) {+ try {+ const jobs = await queue.getJobs([state]);+ for (const job of jobs) {+ if (job.id?.startsWith(`repeat:${schedulerKey}:`)) {+ await job.remove();+ logger.info(+ { jobId: job.id, schedulerKey },+ 'removed conflicting scheduler job'+ );+ }+ }+ } catch {+ // Ignored during cleanup — V1's behaviour (boot-cron.ts:24-26).+ }+ }+}