🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): register recurring cron jobs via atomic BullMQ job schedulers (#16223)
Steffen Schmitz88d250032ab7
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 636b271a3e..b08760d0d1 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -154,6 +154,11 @@ the same PR.- Keep backward compatibility in queue payloads when possible during rollingdeployments.+- Register recurring cron jobs through+ `src/server/redis/scheduleRecurringJob.ts` (BullMQ job schedulers), never+ via the deprecated `Queue.add(name, data, { repeat })` API. When changing a+ cron pattern, append the old pattern to `previousPatterns` so the legacy+ md5-keyed schedule is cleaned up on boot.- Do not hand-edit generated artifacts under `prisma/generated/*` or `dist/*`.- Avoid exposing server-only modules through `src/index.ts` if they must remainfrontend-safe.diff --git a/packages/shared/src/server/redis/blobStorageIntegrationQueue.ts b/packages/shared/src/server/redis/blobStorageIntegrationQueue.tsindex 8ee6d932e3..812ddeb6b8 100644--- a/packages/shared/src/server/redis/blobStorageIntegrationQueue.ts+++ b/packages/shared/src/server/redis/blobStorageIntegrationQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class BlobStorageIntegrationQueue {@@ -35,31 +36,11 @@ export class BlobStorageIntegrationQueue {if (BlobStorageIntegrationQueue.instance) {logger.debug("Scheduling jobs for BlobStorageIntegrationQueue");- // Remove the old hourly cron pattern - BullMQ keys repeatable jobs by- // name + pattern, so changing the pattern creates a second schedule- // while the old one keeps firing.- BlobStorageIntegrationQueue.instance- // eslint-disable-next-line @typescript-eslint/no-deprecated -- Existing repeatable-job cleanup; job scheduler migration should be handled separately.- .removeRepeatable(QueueJobs.BlobStorageIntegrationJob, {- pattern: "20 * * * *",- })- .catch((err) => {- logger.error(- "Error removing legacy BlobStorageIntegrationJob schedule",- err,- );- });- BlobStorageIntegrationQueue.instance- .add(- QueueJobs.BlobStorageIntegrationJob,- {},- {- repeat: { pattern: "*/20 * * * *" }, // every 20 minutes- },- )- .catch((err) => {- logger.error("Error adding BlobStorageIntegrationJob schedule", err);- });+ scheduleRecurringJob(BlobStorageIntegrationQueue.instance, {+ jobName: QueueJobs.BlobStorageIntegrationJob,+ pattern: "*/20 * * * *", // every 20 minutes+ previousPatterns: ["20 * * * *"], // old hourly schedule+ });}return BlobStorageIntegrationQueue.instance;diff --git a/packages/shared/src/server/redis/cloudFreeTierUsageThresholdQueue.ts b/packages/shared/src/server/redis/cloudFreeTierUsageThresholdQueue.tsindex ccf55198d4..6f4a369a31 100644--- a/packages/shared/src/server/redis/cloudFreeTierUsageThresholdQueue.ts+++ b/packages/shared/src/server/redis/cloudFreeTierUsageThresholdQueue.ts@@ -2,6 +2,7 @@ import { Queue } from "bullmq";import { env } from "../../env";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class CloudFreeTierUsageThresholdQueue {@@ -45,20 +46,18 @@ export class CloudFreeTierUsageThresholdQueue {"[CloudFreeTierUsageThresholdQueue] Scheduling recurring job",{pattern: "35 * * * *",- jobId: "free-tier-usage-threshold-hourly",description: "Every hour at minute 35",timestamp: new Date().toISOString(),},);- CloudFreeTierUsageThresholdQueue.instance.add(- QueueJobs.CloudFreeTierUsageThresholdJob,- { type: "recurring" },- {- repeat: { pattern: "35 * * * *" },- // jobId: "free-tier-usage-threshold-hourly", // CRITICAL: Unique ID prevents duplicates across containers- },- );+ // The legacy schedule carried data { type: "recurring" } to distinguish+ // it from a bootstrap job that was never enabled; nothing reads it and+ // job data has no semantics to BullMQ, so the payload is now empty.+ scheduleRecurringJob(CloudFreeTierUsageThresholdQueue.instance, {+ jobName: QueueJobs.CloudFreeTierUsageThresholdJob,+ pattern: "35 * * * *",+ });// Optional: Bootstrap job for immediate execution on startup// This ensures usage thresholds are processed immediately when service startsdiff --git a/packages/shared/src/server/redis/cloudUsageMeteringQueue.ts b/packages/shared/src/server/redis/cloudUsageMeteringQueue.tsindex 7ca0f9b6da..d4a1671ca2 100644--- a/packages/shared/src/server/redis/cloudUsageMeteringQueue.ts+++ b/packages/shared/src/server/redis/cloudUsageMeteringQueue.ts@@ -2,6 +2,7 @@ import { Queue } from "bullmq";import { env } from "../../env";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class CloudUsageMeteringQueue {@@ -41,24 +42,13 @@ export class CloudUsageMeteringQueue {if (CloudUsageMeteringQueue.instance) {logger.info("[CloudUsageMeteringQueue] Scheduling recurring job", {pattern: "5 * * * *",- jobId: "cloud-usage-metering-recurring",timestamp: new Date().toISOString(),});- CloudUsageMeteringQueue.instance- .add(- QueueJobs.CloudUsageMeteringJob,- {},- {- // Run at minute 5 of every hour (e.g. 1:05, 2:05, 3:05, etc)- repeat: { pattern: "5 * * * *" },- },- )- .catch((err) => {- logger.error(- "[CloudUsageMeteringQueue] Failed to schedule recurring job",- err,- );- });+ // Run at minute 5 of every hour (e.g. 1:05, 2:05, 3:05, etc)+ scheduleRecurringJob(CloudUsageMeteringQueue.instance, {+ jobName: QueueJobs.CloudUsageMeteringJob,+ pattern: "5 * * * *",+ });logger.info("[CloudUsageMeteringQueue] Scheduling bootstrap job", {jobId: "cloud-usage-metering-bootstrap",diff --git a/packages/shared/src/server/redis/coreDataS3ExportQueue.ts b/packages/shared/src/server/redis/coreDataS3ExportQueue.tsindex a930801742..bab7fec474 100644--- a/packages/shared/src/server/redis/coreDataS3ExportQueue.ts+++ b/packages/shared/src/server/redis/coreDataS3ExportQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";import { env } from "../../env";@@ -40,17 +41,10 @@ export class CoreDataS3ExportQueue {if (CoreDataS3ExportQueue.instance) {logger.debug("Scheduling jobs for CoreDataS3ExportQueue");- CoreDataS3ExportQueue.instance- .add(- QueueJobs.CoreDataS3ExportJob,- {},- {- repeat: { pattern: "15 3 * * *" }, // every day at 3:15am- },- )- .catch((err) => {- logger.error("Error adding CoreDataS3ExportJob schedule", err);- });+ scheduleRecurringJob(CoreDataS3ExportQueue.instance, {+ jobName: QueueJobs.CoreDataS3ExportJob,+ pattern: "15 3 * * *", // every day at 3:15am+ });}return CoreDataS3ExportQueue.instance;diff --git a/packages/shared/src/server/redis/dataRetentionQueue.ts b/packages/shared/src/server/redis/dataRetentionQueue.tsindex e703fa05f0..eae9969a77 100644--- a/packages/shared/src/server/redis/dataRetentionQueue.ts+++ b/packages/shared/src/server/redis/dataRetentionQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class DataRetentionQueue {@@ -35,17 +36,10 @@ export class DataRetentionQueue {if (DataRetentionQueue.instance) {logger.debug("Scheduling jobs for DataRetentionQueue");- DataRetentionQueue.instance- .add(- QueueJobs.DataRetentionJob,- {},- {- repeat: { pattern: "15 3 * * *" }, // every day at 3:15am- },- )- .catch((err) => {- logger.error("Error adding DataRetentionQueue schedule", err);- });+ scheduleRecurringJob(DataRetentionQueue.instance, {+ jobName: QueueJobs.DataRetentionJob,+ pattern: "15 3 * * *", // every day at 3:15am+ });}return DataRetentionQueue.instance;diff --git a/packages/shared/src/server/redis/dlqRetryQueue.ts b/packages/shared/src/server/redis/dlqRetryQueue.tsindex c50895cc0e..ba109c21b0 100644--- a/packages/shared/src/server/redis/dlqRetryQueue.ts+++ b/packages/shared/src/server/redis/dlqRetryQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class DeadLetterRetryQueue {@@ -35,17 +36,11 @@ export class DeadLetterRetryQueue {if (DeadLetterRetryQueue.instance) {logger.debug("Scheduling jobs for DeadLetterRetryQueue");- DeadLetterRetryQueue.instance- .add(- QueueJobs.DeadLetterRetryJob,- { timestamp: new Date() },- {- repeat: { pattern: "0 */10 * * * *" }, // every 10 minutes (with seconds precision)- },- )- .catch((err) => {- logger.error("Error adding DeadLetterRetryQueue schedule", err);- });+ scheduleRecurringJob(DeadLetterRetryQueue.instance, {+ jobName: QueueJobs.DeadLetterRetryJob,+ pattern: "0 */10 * * * *", // every 10 minutes (with seconds precision)+ data: { timestamp: new Date() },+ });}return DeadLetterRetryQueue.instance;diff --git a/packages/shared/src/server/redis/eventPropagationQueue.ts b/packages/shared/src/server/redis/eventPropagationQueue.tsindex 2622832293..1ee2822510 100644--- a/packages/shared/src/server/redis/eventPropagationQueue.ts+++ b/packages/shared/src/server/redis/eventPropagationQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export class EventPropagationQueue {@@ -43,17 +44,11 @@ export class EventPropagationQueue {});logger.debug("Scheduling jobs for EventPropagationQueue");- EventPropagationQueue.instance- .add(- QueueJobs.EventPropagationJob,- { timestamp: new Date() },- {- repeat: { pattern: "* * * * *" }, // every minute- },- )- .catch((err) => {- logger.error("Error adding EventPropagationQueue schedule", err);- });+ scheduleRecurringJob(EventPropagationQueue.instance, {+ jobName: QueueJobs.EventPropagationJob,+ pattern: "* * * * *", // every minute+ data: { timestamp: new Date() },+ });}return EventPropagationQueue.instance;diff --git a/packages/shared/src/server/redis/meteringDataPostgresExportQueue.ts b/packages/shared/src/server/redis/meteringDataPostgresExportQueue.tsindex e628de863d..b68320ff44 100644--- a/packages/shared/src/server/redis/meteringDataPostgresExportQueue.ts+++ b/packages/shared/src/server/redis/meteringDataPostgresExportQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";import { env } from "../../env";@@ -40,20 +41,10 @@ export class MeteringDataPostgresExportQueue {if (MeteringDataPostgresExportQueue.instance) {logger.debug("Scheduling jobs for MeteringDataPostgresExportQueue");- MeteringDataPostgresExportQueue.instance- .add(- QueueJobs.MeteringDataPostgresExportJob,- {},- {- repeat: { pattern: "30 2 * * *" }, // every day at 2:30am UTC- },- )- .catch((err) => {- logger.error(- "Error adding MeteringDataPostgresExportJob schedule",- err,- );- });+ scheduleRecurringJob(MeteringDataPostgresExportQueue.instance, {+ jobName: QueueJobs.MeteringDataPostgresExportJob,+ pattern: "30 2 * * *", // every day at 2:30am UTC+ });}return MeteringDataPostgresExportQueue.instance;diff --git a/packages/shared/src/server/redis/mixpanelIntegrationQueue.ts b/packages/shared/src/server/redis/mixpanelIntegrationQueue.tsindex ca99284a9f..e95d6c06a0 100644--- a/packages/shared/src/server/redis/mixpanelIntegrationQueue.ts+++ b/packages/shared/src/server/redis/mixpanelIntegrationQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export const MIXPANEL_SYNC_CRON_PATTERN = "30 * * * *"; // every hour at :30@@ -37,17 +38,10 @@ export class MixpanelIntegrationQueue {if (MixpanelIntegrationQueue.instance) {logger.debug("Scheduling jobs for MixpanelIntegrationQueue");- MixpanelIntegrationQueue.instance- .add(- QueueJobs.MixpanelIntegrationJob,- {},- {- repeat: { pattern: MIXPANEL_SYNC_CRON_PATTERN },- },- )- .catch((err) => {- logger.error("Error adding MixpanelIntegrationJob schedule", err);- });+ scheduleRecurringJob(MixpanelIntegrationQueue.instance, {+ jobName: QueueJobs.MixpanelIntegrationJob,+ pattern: MIXPANEL_SYNC_CRON_PATTERN,+ });}return MixpanelIntegrationQueue.instance;diff --git a/packages/shared/src/server/redis/postHogIntegrationQueue.ts b/packages/shared/src/server/redis/postHogIntegrationQueue.tsindex 50fe9f878f..9003077e47 100644--- a/packages/shared/src/server/redis/postHogIntegrationQueue.ts+++ b/packages/shared/src/server/redis/postHogIntegrationQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";export const POSTHOG_SYNC_CRON_PATTERN = "30 * * * *"; // every hour at :30@@ -37,17 +38,10 @@ export class PostHogIntegrationQueue {if (PostHogIntegrationQueue.instance) {logger.debug("Scheduling jobs for PostHogIntegrationQueue");- PostHogIntegrationQueue.instance- .add(- QueueJobs.PostHogIntegrationJob,- {},- {- repeat: { pattern: POSTHOG_SYNC_CRON_PATTERN },- },- )- .catch((err) => {- logger.error("Error adding PostHogIntegrationJob schedule", err);- });+ scheduleRecurringJob(PostHogIntegrationQueue.instance, {+ jobName: QueueJobs.PostHogIntegrationJob,+ pattern: POSTHOG_SYNC_CRON_PATTERN,+ });}return PostHogIntegrationQueue.instance;diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.test.ts b/packages/shared/src/server/redis/scheduleRecurringJob.test.tsnew file mode 100644index 0000000000..17c4cb4816--- /dev/null+++ b/packages/shared/src/server/redis/scheduleRecurringJob.test.ts@@ -0,0 +1,99 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { Queue } from "bullmq";+import { scheduleRecurringJob } from "./scheduleRecurringJob";++vi.mock("../logger", () => ({+ logger: {+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ },+}));++const createQueueMock = () => {+ const removeRepeatable = vi.fn().mockResolvedValue(true);+ const upsertJobScheduler = vi.fn().mockResolvedValue({});+ const queue = {+ name: "test-queue",+ removeRepeatable,+ upsertJobScheduler,+ } as unknown as Queue;+ return { queue, removeRepeatable, upsertJobScheduler };+};++describe("scheduleRecurringJob", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it("removes legacy schedules for the current and previous patterns before upserting", async () => {+ const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "*/15 * * * *",+ previousPatterns: ["25 * * * *"],+ });++ expect(removeRepeatable).toHaveBeenCalledTimes(2);+ expect(removeRepeatable).toHaveBeenCalledWith("my-job", {+ pattern: "*/15 * * * *",+ });+ expect(removeRepeatable).toHaveBeenCalledWith("my-job", {+ pattern: "25 * * * *",+ });+ expect(upsertJobScheduler).toHaveBeenCalledWith(+ "my-job",+ { pattern: "*/15 * * * *" },+ { name: "my-job", data: {} },+ );+ // The legacy cleanup must complete before the scheduler is upserted, so+ // a concurrently created legacy entry cannot outlive the migration.+ for (const removeCall of removeRepeatable.mock.invocationCallOrder) {+ expect(removeCall).toBeLessThan(+ upsertJobScheduler.mock.invocationCallOrder[0]!,+ );+ }+ });++ it("passes template data through to the scheduler", async () => {+ const { queue, upsertJobScheduler } = createQueueMock();++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "* * * * *",+ data: { type: "recurring" },+ });++ expect(upsertJobScheduler).toHaveBeenCalledWith(+ "my-job",+ { pattern: "* * * * *" },+ { name: "my-job", data: { type: "recurring" } },+ );+ });++ it("still upserts the scheduler when legacy cleanup fails", async () => {+ const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();+ removeRepeatable.mockRejectedValue(new Error("redis down"));++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "* * * * *",+ });++ expect(upsertJobScheduler).toHaveBeenCalledTimes(1);+ });++ it("resolves instead of rejecting when the upsert fails", async () => {+ const { queue, upsertJobScheduler } = createQueueMock();+ upsertJobScheduler.mockRejectedValue(new Error("redis down"));++ await expect(+ scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "* * * * *",+ }),+ ).resolves.toBeUndefined();+ });+});diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.ts b/packages/shared/src/server/redis/scheduleRecurringJob.tsnew file mode 100644index 0000000000..ef31ecff67--- /dev/null+++ b/packages/shared/src/server/redis/scheduleRecurringJob.ts@@ -0,0 +1,78 @@+import type { Queue } from "bullmq";+import { logger } from "../logger";++// DO NOT REMOVE THIS FILE AND BEHAVIOUR WITHIN A MINOR LANGFUSE VERSION.+// THE CLEANUP BELOW DELETES THE OLD md5-KEYED REPEAT SCHEDULES; WITHOUT IT,+// SELF-HOSTERS UPGRADING FROM A LEGACY-SCHEDULING VERSION GET DUPLICATE CRON+// TRIGGERS FOREVER (bullmq v6 workers keep iterating stranded md5-keyed+// chains; only manual Redis cleanup stops them).+// HOLD THE BULLMQ v6 UPGRADE UNTIL LANGFUSE V5: v6 deletes+// Queue.removeRepeatable(), which this cleanup requires.++/**+ * Registers a recurring cron job on a queue via a BullMQ job scheduler.+ *+ * Job schedulers replace the deprecated `Queue.add(name, data, { repeat })`+ * API. The legacy path replaced the pending next iteration in two separate+ * Redis round trips on every boot (delete the delayed job, then re-create+ * it), so a producer failing between the two - e.g. a container killed+ * mid-boot during a fleet-wide deploy - silently ended the schedule until+ * the next boot re-registered it. `upsertJobScheduler` performs the same+ * replacement in a single atomic Lua script, so concurrent boots and+ * mid-boot crashes cannot lose the chain. BullMQ v6 removes the legacy API+ * entirely.+ *+ * The scheduler id is the job name. Legacy schedules are keyed by+ * md5(name + pattern) instead, so this helper also removes the legacy+ * entries for the current pattern and any historical ones - without that+ * cleanup the old chain keeps firing in parallel with the scheduler. The+ * cleanup is idempotent and must stay in place for as long as a direct+ * upgrade from a Langfuse version that scheduled via the legacy API is+ * supported.+ *+ * Never rejects: scheduling runs fire-and-forget from queue singleton+ * getters, so all failures are logged instead of thrown.+ */+export const scheduleRecurringJob = async (+ queue: Queue,+ opts: {+ jobName: string;+ pattern: string;+ /**+ * Cron patterns this job was registered with via the legacy repeat API+ * in earlier releases. The current pattern is always cleaned up and does+ * not need to be listed. When changing `pattern`, append the old value+ * here, since each legacy pattern was a separate schedule.+ */+ previousPatterns?: string[];+ /** Template data stored on the scheduler and passed to every iteration. */+ data?: Record<string, unknown>;+ },+): Promise<void> => {+ const { jobName, pattern, previousPatterns = [], data } = opts;++ for (const legacyPattern of [pattern, ...previousPatterns]) {+ try {+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- Removes schedules created by the deprecated repeat API in earlier releases; keep while direct upgrades from those releases are supported.+ await queue.removeRepeatable(jobName, { pattern: legacyPattern });+ } catch (err) {+ logger.error(+ `Error removing legacy ${jobName} schedule (${legacyPattern}) on ${queue.name}`,+ err,+ );+ }+ }++ try {+ await queue.upsertJobScheduler(+ jobName,+ { pattern },+ { name: jobName, data: data ?? {} },+ );+ } catch (err) {+ logger.error(+ `Error upserting ${jobName} scheduler on ${queue.name} - the recurring job may not run until the next boot`,+ err,+ );+ }+};diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.tsindex ea212184eb..b4e1130054 100644--- a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts@@ -12,15 +12,15 @@ describe("V4LegacyApiUsageQueue schedule", () => {vi.clearAllMocks();});- it("removes the legacy hourly repeatable job before scheduling */15", async () => {- const removeRepeatable = vi.fn().mockResolvedValue(undefined);- const add = vi.fn().mockResolvedValue(undefined);+ it("removes the legacy repeatable schedules and upserts a job scheduler", async () => {+ const removeRepeatable = vi.fn().mockResolvedValue(true);+ const upsertJobScheduler = vi.fn().mockResolvedValue({});const on = vi.fn();vi.doMock("bullmq", () => ({Queue: class {removeRepeatable = removeRepeatable;- add = add;+ upsertJobScheduler = upsertJobScheduler;on = on;},}));@@ -45,18 +45,31 @@ describe("V4LegacyApiUsageQueue schedule", () => {V4LegacyApiUsageQueue.getInstance();+ // Scheduling is fire-and-forget from getInstance, so wait for the chain.+ await vi.waitFor(() => {+ expect(upsertJobScheduler).toHaveBeenCalledTimes(1);+ });++ // Legacy repeatable entries are keyed by md5(name + pattern), so both the+ // current pattern and the pre-migration hourly pattern must be cleaned up.+ expect(removeRepeatable).toHaveBeenCalledWith(+ QueueJobs.V4LegacyApiUsageJob,+ { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },+ );expect(removeRepeatable).toHaveBeenCalledWith(QueueJobs.V4LegacyApiUsageJob,{ pattern: "25 * * * *" },);- expect(add).toHaveBeenCalledWith(+ expect(upsertJobScheduler).toHaveBeenCalledWith(QueueJobs.V4LegacyApiUsageJob,- {},- { repeat: { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN } },+ { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },+ { name: QueueJobs.V4LegacyApiUsageJob, data: {} },);expect(V4_LEGACY_API_USAGE_CRON_PATTERN).toBe("*/15 * * * *");- expect(removeRepeatable.mock.invocationCallOrder[0]).toBeLessThan(- add.mock.invocationCallOrder[0]!,- );+ for (const removeCall of removeRepeatable.mock.invocationCallOrder) {+ expect(removeCall).toBeLessThan(+ upsertJobScheduler.mock.invocationCallOrder[0]!,+ );+ }});});diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.tsindex 106c990584..1b13eeb1a4 100644--- a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.ts@@ -1,6 +1,7 @@import { Queue } from "bullmq";import { QueueName, QueueJobs } from "../queues";import { createBullMQQueueOptionsWithRedis } from "./redis";+import { scheduleRecurringJob } from "./scheduleRecurringJob";import { logger } from "../logger";// Every 15 minutes. The worker re-scans a trailing margin each run, so the@@ -39,31 +40,11 @@ export class V4LegacyApiUsageQueue {if (V4LegacyApiUsageQueue.instance) {logger.debug("Scheduling jobs for V4LegacyApiUsageQueue");- // Remove the old hourly cron pattern - BullMQ keys repeatable jobs by- // name + pattern, so changing the pattern creates a second schedule- // while the old one keeps firing.- V4LegacyApiUsageQueue.instance- // eslint-disable-next-line @typescript-eslint/no-deprecated -- Existing repeatable-job cleanup; job scheduler migration should be handled separately.- .removeRepeatable(QueueJobs.V4LegacyApiUsageJob, {- pattern: "25 * * * *",- })- .catch((err) => {- logger.error(- "Error removing legacy V4LegacyApiUsageJob schedule",- err,- );- });- V4LegacyApiUsageQueue.instance- .add(- QueueJobs.V4LegacyApiUsageJob,- {},- {- repeat: { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },- },- )- .catch((err) => {- logger.error("Error adding V4LegacyApiUsageJob schedule", err);- });+ scheduleRecurringJob(V4LegacyApiUsageQueue.instance, {+ jobName: QueueJobs.V4LegacyApiUsageJob,+ pattern: V4_LEGACY_API_USAGE_CRON_PATTERN,+ previousPatterns: ["25 * * * *"], // old hourly schedule+ });}return V4LegacyApiUsageQueue.instance;