🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify job handling
Edward Cuif0516615be36
diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.tsindex dd4debe8a..24c9bbe88 100644--- a/server/src/queues/adapters/bullMQAdapter.ts+++ b/server/src/queues/adapters/bullMQAdapter.ts@@ -1,5 +1,5 @@import { Job, Queue, QueueEvents, Worker } from "bullmq";-import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";+import { IJobQueue } from "../jobQueue.js";export class BullMQAdapter implements IJobQueue {private readonly connection: { host: string; port: number; password?: string };@@ -78,7 +78,6 @@ export class BullMQAdapter implements IJobQueue {let queueEvents: QueueEvents | undefined;try {- // Create Queuequeue = new Queue(queueName, {connection: this.connection,defaultJobOptions: {@@ -90,14 +89,12 @@ export class BullMQAdapter implements IJobQueue {this.queues.set(queueName, queue);- // Create QueueEvents for monitoringqueueEvents = new QueueEvents(queueName, {connection: this.connection,});this.queueEvents.set(queueName, queueEvents);- // Log job eventsqueueEvents.on("completed", ({ jobId }) => {console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);});@@ -108,7 +105,6 @@ export class BullMQAdapter implements IJobQueue {} catch (error) {console.error(`[BullMQ] Failed to create queue ${queueName}:`, error);- // Clean up any partially created resourcesconst cleanupPromises: Promise<void>[] = [];if (queueEvents) {@@ -129,7 +125,6 @@ export class BullMQAdapter implements IJobQueue {);}- // Wait for cleanup to complete before rethrowingawait Promise.all(cleanupPromises);throw error;@@ -137,20 +132,14 @@ export class BullMQAdapter implements IJobQueue {}}- async send<T>(queueName: string, data: T): Promise<string> {+ async send<T>(queueName: string, data: T): Promise<void> {const queue = this.queues.get(queueName);if (!queue) {throw new Error(`Queue ${queueName} not found. Call createQueue first.`);}try {- const job = await queue.add(queueName, data);-- if (!job.id) {- throw new Error(`Failed to enqueue job to ${queueName}`);- }-- return job.id;+ await queue.add(queueName, data);} catch (error) {const errorMessage = error instanceof Error ? error.message : String(error);console.error(`[BullMQ] Failed to enqueue job to ${queueName}:`, error);@@ -158,31 +147,16 @@ export class BullMQAdapter implements IJobQueue {}}- async work<T>(- queueName: string,- config: JobConfig,- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>- ): Promise<void> {+ async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {try {const worker = new Worker(queueName,async (job: Job<T>) => {- const normalizedJob: JobData<T> = {- id: job.id!,- data: job.data,- };-- // NOTE: Interface Impedance Mismatch- // BullMQ processes one job at a time per worker, while pg-boss can batch jobs.- // The IJobQueue interface expects handlers to accept arrays to support pg-boss batching.- // Here we wrap single BullMQ jobs in an array to maintain interface compatibility.- // This is an acceptable tradeoff: pg-boss gets true batching, BullMQ gets array of 1 item.- await handler([normalizedJob]);+ await handler(job.data);},{connection: this.connection,- concurrency: config.concurrency ?? config.batchSize ?? 1,- limiter: config.limiter,+ concurrency: 1,});@@ -196,11 +170,7 @@ export class BullMQAdapter implements IJobQueue {this.workers.set(queueName, worker);} catch (error) {- console.error(- `[BullMQ] Failed to create worker for queue ${queueName} with config:`,- { concurrency: config.concurrency ?? config.batchSize ?? 1, limiter: config.limiter },- error- );+ console.error(`[BullMQ] Failed to create worker for queue ${queueName}`, error);throw error;}}diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.tsindex cc420ff12..780c445c3 100644--- a/server/src/queues/adapters/pgBossAdapter.ts+++ b/server/src/queues/adapters/pgBossAdapter.ts@@ -1,5 +1,5 @@import PgBoss, { Job } from "pg-boss";-import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";+import { IJobQueue } from "../jobQueue.js";export class PgBossAdapter implements IJobQueue {private boss: PgBoss;@@ -35,34 +35,19 @@ export class PgBossAdapter implements IJobQueue {await this.boss.createQueue(queueName);}- async send<T>(queueName: string, data: T): Promise<string> {- const jobId = await this.boss.send(queueName, data as object);-- if (!jobId) {- throw new Error(`Failed to enqueue job to ${queueName}`);- }-- return jobId;+ async send<T>(queueName: string, data: T): Promise<void> {+ await this.boss.send(queueName, data as object);}- async work<T>(- queueName: string,- config: JobConfig,- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>- ): Promise<void> {+ async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {await this.boss.work(queueName,{- batchSize: config.batchSize ?? 1,- pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,+ batchSize: 1,+ pollingIntervalSeconds: 3,},- async (jobs: Job<T>[]) => {- const normalizedJobs: JobData<T>[] = jobs.map(job => ({- id: job.id,- data: job.data,- }));-- await handler(normalizedJobs);+ async ([job]: Job<T>[]) => {+ await handler(job.data);});}diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.tsindex f47e7ef3e..b1fa693cc 100644--- a/server/src/queues/jobQueue.ts+++ b/server/src/queues/jobQueue.ts@@ -1,49 +1,7 @@-export interface JobConfig {- /**- * Number of jobs to process in a batch (pg-boss specific)- * For BullMQ, this maps to concurrency- */- batchSize?: number;-- /**- * Polling interval in seconds (pg-boss specific)- * BullMQ uses event-driven model and ignores this- */- pollingIntervalSeconds?: number;-- /**- * Number of concurrent workers (BullMQ specific)- * Falls back to batchSize if not specified- */- concurrency?: number;-- /**- * Rate limiting configuration (BullMQ specific)- */- limiter?: {- max: number;- duration: number;- };-}--export interface JobResult {- success: boolean;- error?: Error;-}--export interface JobData<T> {- id: string;- data: T;-}-export interface IJobQueue {start(): Promise<void>;stop(): Promise<void>;createQueue(queueName: string): Promise<void>;- send<T>(queueName: string, data: T): Promise<string>;- work<T>(- queueName: string,- config: JobConfig,- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>- ): Promise<void>;+ send<T>(queueName: string, data: T): Promise<void>;+ work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;}diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 1782fb9da..fe1038d34 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -78,8 +78,8 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {export async function registerCsvParseWorker() {const jobQueue = getJobQueue();- await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {- const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;+ await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {+ const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;let stream: ReturnType<typeof parse> | null = null;let processingTimeout: NodeJS.Timeout | null = null;diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex f96ea2e0a..407242af4 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -16,8 +16,8 @@ const getImportDataMapping = (platform: string) => {export async function registerDataInsertWorker() {const jobQueue = getJobQueue();- await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {- const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;+ await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {+ const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job;// Handle finalization signalif (allChunksSent) {