Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(P2-005): jobs/schedulers.ts: the boot-cron port, all 20 ids byte-identical

ralphb92f6e1cb3f8
diff --git a/packages/core/src/jobs/schedulers.test.ts b/packages/core/src/jobs/schedulers.test.ts
new file mode 100644
index 000000000..c276c2922
--- /dev/null
+++ b/packages/core/src/jobs/schedulers.test.ts
@@ -0,0 +1,293 @@
+import { expect, test } from 'bun:test';
+import type { Logger } from '../logger';
+import {
+ CRON_SCHEDULES,
+ PING_SCHEDULE,
+ type SchedulerQueue,
+ startSchedulers,
+} from './schedulers';
+
+// Byte-identity snapshot, mirroring
+// `verification/golden/queue-keys/scheduler-ids.json` — which the
+// controller's `check.sh` independently recomputes from V1's
+// `packages/queue/src/queues.ts` (CronQueuePayload union) and
+// `apps/worker/src/boot-cron.ts`. Both sides pin the same 20 ids, including
+// the three ADR-005 fixed omissions (ADR-005 acceptance note).
+const GOLDEN_SCHEDULER_IDS = [
+ 'cohortRefresh',
+ 'dataHealth',
+ 'delete',
+ 'flushEvents',
+ 'flushExports',
+ 'flushGroups',
+ 'flushProfileBackfill',
+ 'flushProfiles',
+ 'flushReplay',
+ 'flushSessions',
+ 'gscSync',
+ 'insightCleanup',
+ 'insightsDaily',
+ 'onboarding',
+ 'ping',
+ 'salt',
+ 'sessionReaper',
+ 'sessionVacuum',
+ 'weeklyDigest',
+ 'windDown',
+].sort();
+
+function stubLogger(): Logger {
+ const noop = () => undefined;
+ const logger: Logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+}
+
+/** In-memory stand-in for a BullMQ queue's scheduler surface. */
+function fakeQueue(options?: {
+ existingSchedulerKeys?: string[];
+ conflictOnce?: Set<string>;
+ conflictingJobs?: { id: string }[];
+}): SchedulerQueue & {
+ calls: {
+ upserted: { id: string; data: unknown }[];
+ removedSchedulers: string[];
+ removedJobs: string[];
+ };
+} {
+ const schedulerKeys = new Set(options?.existingSchedulerKeys ?? []);
+ const conflictOnce = new Set(options?.conflictOnce ?? []);
+ const remainingConflictingJobs = [...(options?.conflictingJobs ?? [])];
+ const calls = {
+ upserted: [] as { id: string; data: unknown }[],
+ removedSchedulers: [] as string[],
+ removedJobs: [] as string[],
+ };
+
+ return {
+ calls,
+ async getJobSchedulers() {
+ return [...schedulerKeys].map((key) => ({ key }));
+ },
+ async upsertJobScheduler(id, _repeatOpts, jobTemplate) {
+ if (conflictOnce.has(id)) {
+ conflictOnce.delete(id);
+ throw new Error('Job scheduler job ID already exists');
+ }
+ schedulerKeys.add(id);
+ calls.upserted.push({ id, data: jobTemplate?.data });
+ return undefined;
+ },
+ async removeJobScheduler(id) {
+ calls.removedSchedulers.push(id);
+ return schedulerKeys.delete(id);
+ },
+ async getJobs(_types) {
+ return remainingConflictingJobs.splice(0).map((job) => ({
+ id: job.id,
+ remove: async () => {
+ calls.removedJobs.push(job.id);
+ },
+ }));
+ },
+ };
+}
+
+test('the golden scheduler ids are reproduced byte-for-byte', () => {
+ const allIds = [...CRON_SCHEDULES.map((s) => s.id), PING_SCHEDULE.id].sort();
+
+ expect(allIds).toHaveLength(20);
+ expect(allIds).toEqual(GOLDEN_SCHEDULER_IDS);
+});
+
+test('the three ADR-005 fixed omissions are present', () => {
+ const ids = new Set(CRON_SCHEDULES.map((s) => s.id));
+ expect(ids.has('dataHealth')).toBe(true);
+ expect(ids.has('windDown')).toBe(true);
+ expect(ids.has('flushExports')).toBe(true);
+});
+
+test('ping is excluded by default — neither self-hosted nor production', async () => {
+ const queue = fakeQueue();
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ });
+
+ const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();
+ expect(upsertedIds).toHaveLength(19);
+ expect(upsertedIds).not.toContain('ping');
+});
+
+test('ping is excluded when self-hosted but not production', async () => {
+ const queue = fakeQueue();
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: true, production: false },
+ logger: stubLogger(),
+ });
+
+ expect(queue.calls.upserted.map((u) => u.id)).not.toContain('ping');
+});
+
+test('ping is excluded when production but not self-hosted', async () => {
+ const queue = fakeQueue();
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: true },
+ logger: stubLogger(),
+ });
+
+ expect(queue.calls.upserted.map((u) => u.id)).not.toContain('ping');
+});
+
+test('ping is included only when self-hosted AND production', async () => {
+ const queue = fakeQueue();
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: true, production: true },
+ logger: stubLogger(),
+ });
+
+ const upsertedIds = queue.calls.upserted.map((u) => u.id).sort();
+ expect(upsertedIds).toHaveLength(20);
+ expect(upsertedIds).toEqual(GOLDEN_SCHEDULER_IDS);
+});
+
+test('every upsert carries envelope-shaped data, not a legacy {type} shape', async () => {
+ const queue = fakeQueue();
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],
+ });
+
+ expect(queue.calls.upserted).toEqual([
+ { id: 'salt', data: { payload: null, meta: {} } },
+ ]);
+});
+
+test('a scheduler no longer declared is pruned', async () => {
+ const queue = fakeQueue({
+ existingSchedulerKeys: ['salt', 'aRemovedJob'],
+ });
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],
+ });
+
+ expect(queue.calls.removedSchedulers).toEqual(['aRemovedJob']);
+});
+
+test('a declared scheduler that already exists is left alone by pruning', async () => {
+ const queue = fakeQueue({ existingSchedulerKeys: ['salt', 'delete'] });
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [
+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },
+ { id: 'delete', schedule: { pattern: '0 * * * *' } },
+ ],
+ });
+
+ expect(queue.calls.removedSchedulers).toEqual([]);
+});
+
+test('a conflicting upsert is cleaned up and retried, not left failed', async () => {
+ const queue = fakeQueue({
+ conflictOnce: new Set(['salt']),
+ conflictingJobs: [{ id: 'repeat:salt:1700000000000' }],
+ });
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],
+ });
+
+ // Cleanup removed the stale scheduler-created job and the scheduler
+ // itself, then the retry succeeded.
+ expect(queue.calls.removedJobs).toEqual(['repeat:salt:1700000000000']);
+ expect(queue.calls.removedSchedulers).toContain('salt');
+ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['salt']);
+});
+
+test('a conflicting job belonging to a different scheduler is left alone', async () => {
+ const queue = fakeQueue({
+ conflictOnce: new Set(['salt']),
+ conflictingJobs: [{ id: 'repeat:delete:1700000000000' }],
+ });
+
+ await startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [{ id: 'salt', schedule: { pattern: '0 0 * * *' } }],
+ });
+
+ expect(queue.calls.removedJobs).toEqual([]);
+ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['salt']);
+});
+
+test('a non-conflict upsert error is logged, not thrown — one bad scheduler must not block the rest', async () => {
+ const queue = fakeQueue();
+ const originalUpsert = queue.upsertJobScheduler.bind(queue);
+ queue.upsertJobScheduler = async (id, repeatOpts, jobTemplate) => {
+ if (id === 'salt') {
+ throw new Error('ECONNRESET');
+ }
+ return originalUpsert(id, repeatOpts, jobTemplate);
+ };
+
+ await expect(
+ startSchedulers({
+ queue,
+ flags: { selfHosted: false, production: false },
+ logger: stubLogger(),
+ schedulers: [
+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },
+ { id: 'delete', schedule: { pattern: '0 * * * *' } },
+ ],
+ })
+ ).resolves.toBeUndefined();
+
+ expect(queue.calls.upserted.map((u) => u.id)).toEqual(['delete']);
+});
+
+test('idempotent across replicas — a second boot converges on the same set, no duplicates', async () => {
+ const queue = fakeQueue();
+ const schedulers = [
+ { id: 'salt', schedule: { pattern: '0 0 * * *' } as const },
+ { id: 'delete', schedule: { pattern: '0 * * * *' } as const },
+ ];
+ const flags = { selfHosted: false, production: false };
+
+ await startSchedulers({ queue, flags, logger: stubLogger(), schedulers });
+ await startSchedulers({ queue, flags, logger: stubLogger(), schedulers });
+
+ const finalKeys = (await queue.getJobSchedulers()).map((s) => s.key).sort();
+ expect(finalKeys).toEqual(['delete', 'salt']);
+ // Neither run finds a scheduler outside its declared set, so pruning never
+ // fires — confirms a second replica's boot neither drops nor duplicates.
+ expect(queue.calls.removedSchedulers).toEqual([]);
+});
diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.ts
new file mode 100644
index 000000000..f6894ad83
--- /dev/null
+++ b/packages/core/src/jobs/schedulers.ts
@@ -0,0 +1,228 @@
+// Ports apps/worker/src/boot-cron.ts onto BullMQ's job-scheduler API
+// (ADR-005). Upsert is keyed on job name — the V1 scheduler id — which is
+// what makes it idempotent across replicas: two processes upserting the same
+// id with the same repeat options at boot converge on one scheduler, with no
+// locking of our own needed.
+
+import type { Logger } from '../logger';
+import { wrap } from './envelope';
+
+export type RepeatSchedule = { pattern: string } | { every: number };
+
+export interface SchedulerDefinition {
+ /** V1 scheduler id: the job-scheduler key, and — via BullMQ's own default
+ * (`jobName ?? jobSchedulerId`) — the job name each run dispatches on. */
+ id: string;
+ schedule: RepeatSchedule;
+}
+
+export interface SchedulerFlags {
+ selfHosted: boolean;
+ production: boolean;
+}
+
+/**
+ * The slice of BullMQ's `Queue` this needs, narrowed so a test can fake it
+ * with no Redis connection — mirrors `RunnableJob` in `workers.ts`.
+ */
+export interface SchedulerQueue {
+ getJobSchedulers(): Promise<{ key: string }[]>;
+ upsertJobScheduler(
+ id: string,
+ repeatOpts: RepeatSchedule,
+ jobTemplate?: { data?: unknown }
+ ): Promise<unknown>;
+ removeJobScheduler(id: string): Promise<boolean>;
+ getJobs(
+ types: readonly ('delayed' | 'waiting' | 'completed' | 'failed')[]
+ ): Promise<{ id?: string | null; remove(): Promise<void> }[]>;
+}
+
+const CONFLICT_JOB_STATES = [
+ 'delayed',
+ 'waiting',
+ 'completed',
+ 'failed',
+] as const;
+
+const CONFLICT_ERROR_SUBSTRING = 'job ID already exists';
+
+const MINUTE_MS = 60_000;
+
+// V1's exact 19 always-on scheduler ids and cadences
+// (apps/worker/src/boot-cron.ts). `ping` is the 20th and is conditional —
+// see `PING_SCHEDULE` and `startSchedulers`.
+export const CRON_SCHEDULES: readonly SchedulerDefinition[] = [
+ { id: 'salt', schedule: { pattern: '0 0 * * *' } },
+ { id: 'delete', schedule: { pattern: '0 * * * *' } },
+ { id: 'flushEvents', schedule: { every: 10_000 } },
+ { id: 'flushProfiles', schedule: { every: 10_000 } },
+ { id: 'flushSessions', schedule: { every: 10_000 } },
+ { id: 'flushProfileBackfill', schedule: { every: 30_000 } },
+ { id: 'flushReplay', schedule: { every: 10_000 } },
+ { id: 'flushGroups', schedule: { every: 10_000 } },
+ { id: 'insightsDaily', schedule: { pattern: '0 2 * * *' } },
+ { id: 'onboarding', schedule: { pattern: '0 * * * *' } },
+ { id: 'gscSync', schedule: { pattern: '0 3 * * *' } },
+ { id: 'cohortRefresh', schedule: { pattern: '*/30 * * * *' } },
+ { id: 'sessionReaper', schedule: { every: 5 * MINUTE_MS } },
+ // Daily 04:00 UTC — backstop for cleanup leaks.
+ { id: 'sessionVacuum', schedule: { pattern: '0 4 * * *' } },
+ // Daily 04:30 UTC — prunes stale insights/events.
+ { id: 'insightCleanup', schedule: { pattern: '30 4 * * *' } },
+ // Mondays 08:00 UTC — weekly analytics digest email.
+ { id: 'weeklyDigest', schedule: { pattern: '0 8 * * 1' } },
+ // Daily 07:30 UTC — no-data / data-stopped rescue emails. Owner: misc
+ // (ADR-005 acceptance note completes the ownership map).
+ { id: 'dataHealth', schedule: { pattern: '30 7 * * *' } },
+ // Hourly — expired-trial wind-down emails, block, delete. Owner: organization.
+ { id: 'windDown', schedule: { pattern: '0 * * * *' } },
+ // Every 1 minute — drains export buffers to S3/GCS. Owner: integration.
+ { id: 'flushExports', schedule: { every: MINUTE_MS } },
+];
+
+// V1 gated this on `SELF_HOSTED && NODE_ENV === 'production'`
+// (apps/worker/src/boot-cron.ts:133).
+export const PING_SCHEDULE: SchedulerDefinition = {
+ id: 'ping',
+ schedule: { pattern: '0 0 * * *' },
+};
+
+export interface StartSchedulersOptions {
+ queue: SchedulerQueue;
+ flags: SchedulerFlags;
+ logger: Logger;
+ /** Defaults to the V1 cron set; overridable so a test can use a small fixture. */
+ schedulers?: readonly SchedulerDefinition[];
+}
+
+/**
+ * Upserts every declared scheduler, keyed on job name, and prunes any
+ * scheduler in Redis that is no longer declared — so an owner removed from
+ * `schedulers` stops running rather than lingering forever.
+ *
+ * Safe to call from every replica at boot: `upsertJobScheduler` is BullMQ's
+ * own idempotent operation, and pruning only ever removes schedulers this
+ * call does not want, never ones another replica is mid-upserting.
+ */
+export async function startSchedulers({
+ queue,
+ flags,
+ logger,
+ schedulers = CRON_SCHEDULES,
+}: StartSchedulersOptions): Promise<void> {
+ const desired =
+ flags.selfHosted && flags.production
+ ? [...schedulers, PING_SCHEDULE]
+ : schedulers;
+
+ logger.info('updating cron jobs');
+
+ await pruneStaleSchedulers(queue, new Set(desired.map((s) => s.id)), logger);
+
+ for (const scheduler of desired) {
+ await upsertWithConflictRetry(queue, scheduler, logger);
+ }
+}
+
+async function pruneStaleSchedulers(
+ queue: SchedulerQueue,
+ desiredIds: Set<string>,
+ logger: Logger
+): Promise<void> {
+ const current = await queue.getJobSchedulers().catch((error) => {
+ logger.error({ err: error }, 'error getting job schedulers');
+ return [];
+ });
+
+ for (const scheduler of current) {
+ if (desiredIds.has(scheduler.key)) {
+ continue;
+ }
+ await queue.removeJobScheduler(scheduler.key).catch((error) => {
+ logger.error(
+ { err: error, jobScheduler: scheduler.key },
+ 'error removing job scheduler'
+ );
+ });
+ }
+}
+
+async function upsertWithConflictRetry(
+ queue: SchedulerQueue,
+ scheduler: SchedulerDefinition,
+ logger: Logger
+): Promise<void> {
+ try {
+ await upsert(queue, scheduler);
+ } catch (error) {
+ if (!isConflictError(error)) {
+ logger.error(
+ { err: error, job: scheduler.id },
+ 'error upserting job scheduler'
+ );
+ return;
+ }
+
+ logger.warn(
+ { job: scheduler.id },
+ 'job scheduler conflict detected, attempting cleanup'
+ );
+ await removeConflictingJobs(queue, scheduler.id, logger);
+ // Also try removing the scheduler itself to start fresh.
+ await queue.removeJobScheduler(scheduler.id).catch(() => undefined);
+
+ try {
+ await upsert(queue, scheduler);
+ logger.info({ job: scheduler.id }, 'job scheduler created after cleanup');
+ } catch (retryError) {
+ logger.error(
+ { err: retryError, job: scheduler.id },
+ 'error upserting job scheduler after cleanup'
+ );
+ }
+ }
+}
+
+function upsert(
+ queue: SchedulerQueue,
+ scheduler: SchedulerDefinition
+): Promise<unknown> {
+ // Envelope-shaped for every scheduler-created job — these are freshly
+ // enqueued, never a replay of a V1-shaped job, so there is nothing for the
+ // `cron` compat hook to do here (ADR-005: new data only, old shape read
+ // back through `resolveJob`).
+ return queue.upsertJobScheduler(scheduler.id, scheduler.schedule, {
+ data: wrap(null, {}),
+ });
+}
+
+function isConflictError(error: unknown): boolean {
+ return (
+ error instanceof Error && error.message.includes(CONFLICT_ERROR_SUBSTRING)
+ );
+}
+
+/** BullMQ scheduler-created job ids look like `repeat:<key>:<timestamp>`. */
+async function removeConflictingJobs(
+ queue: SchedulerQueue,
+ schedulerKey: string,
+ logger: Logger
+): Promise<void> {
+ for (const state of CONFLICT_JOB_STATES) {
+ try {
+ const jobs = await queue.getJobs([state]);
+ for (const job of jobs) {
+ if (job.id?.startsWith(`repeat:${schedulerKey}:`)) {
+ await job.remove();
+ logger.info(
+ { jobId: job.id, schedulerKey },
+ 'removed conflicting scheduler job'
+ );
+ }
+ }
+ } catch {
+ // Ignored during cleanup — V1's behaviour (boot-cron.ts:24-26).
+ }
+ }
+}