Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify job handling

Edward Cuif0516615be36
diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.ts
index dd4debe8a..24c9bbe88 100644
--- a/server/src/queues/adapters/bullMQAdapter.ts
+++ b/server/src/queues/adapters/bullMQAdapter.ts
@@ -1,5 +1,5 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
-import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";
+import { IJobQueue } from "../jobQueue.js";
export class BullMQAdapter implements IJobQueue {
private readonly connection: { host: string; port: number; password?: string };
@@ -78,7 +78,6 @@ export class BullMQAdapter implements IJobQueue {
let queueEvents: QueueEvents | undefined;
try {
- // Create Queue
queue = new Queue(queueName, {
connection: this.connection,
defaultJobOptions: {
@@ -90,14 +89,12 @@ export class BullMQAdapter implements IJobQueue {
this.queues.set(queueName, queue);
- // Create QueueEvents for monitoring
queueEvents = new QueueEvents(queueName, {
connection: this.connection,
});
this.queueEvents.set(queueName, queueEvents);
- // Log job events
queueEvents.on("completed", ({ jobId }) => {
console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);
});
@@ -108,7 +105,6 @@ export class BullMQAdapter implements IJobQueue {
} catch (error) {
console.error(`[BullMQ] Failed to create queue ${queueName}:`, error);
- // Clean up any partially created resources
const cleanupPromises: Promise<void>[] = [];
if (queueEvents) {
@@ -129,7 +125,6 @@ export class BullMQAdapter implements IJobQueue {
);
}
- // Wait for cleanup to complete before rethrowing
await Promise.all(cleanupPromises);
throw error;
@@ -137,20 +132,14 @@ export class BullMQAdapter implements IJobQueue {
}
}
- async send<T>(queueName: string, data: T): Promise<string> {
+ async send<T>(queueName: string, data: T): Promise<void> {
const queue = this.queues.get(queueName);
if (!queue) {
throw new Error(`Queue ${queueName} not found. Call createQueue first.`);
}
try {
- const job = await queue.add(queueName, data);
-
- if (!job.id) {
- throw new Error(`Failed to enqueue job to ${queueName}`);
- }
-
- return job.id;
+ await queue.add(queueName, data);
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.error(`[BullMQ] Failed to enqueue job to ${queueName}:`, error);
@@ -158,31 +147,16 @@ export class BullMQAdapter implements IJobQueue {
}
}
- async work<T>(
- queueName: string,
- config: JobConfig,
- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
- ): Promise<void> {
+ async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {
try {
const worker = new Worker(
queueName,
async (job: Job<T>) => {
- const normalizedJob: JobData<T> = {
- id: job.id!,
- data: job.data,
- };
-
- // NOTE: Interface Impedance Mismatch
- // BullMQ processes one job at a time per worker, while pg-boss can batch jobs.
- // The IJobQueue interface expects handlers to accept arrays to support pg-boss batching.
- // Here we wrap single BullMQ jobs in an array to maintain interface compatibility.
- // This is an acceptable tradeoff: pg-boss gets true batching, BullMQ gets array of 1 item.
- await handler([normalizedJob]);
+ await handler(job.data);
},
{
connection: this.connection,
- concurrency: config.concurrency ?? config.batchSize ?? 1,
- limiter: config.limiter,
+ concurrency: 1,
}
);
@@ -196,11 +170,7 @@ export class BullMQAdapter implements IJobQueue {
this.workers.set(queueName, worker);
} catch (error) {
- console.error(
- `[BullMQ] Failed to create worker for queue ${queueName} with config:`,
- { concurrency: config.concurrency ?? config.batchSize ?? 1, limiter: config.limiter },
- error
- );
+ console.error(`[BullMQ] Failed to create worker for queue ${queueName}`, error);
throw error;
}
}
diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.ts
index cc420ff12..780c445c3 100644
--- a/server/src/queues/adapters/pgBossAdapter.ts
+++ b/server/src/queues/adapters/pgBossAdapter.ts
@@ -1,5 +1,5 @@
import PgBoss, { Job } from "pg-boss";
-import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";
+import { IJobQueue } from "../jobQueue.js";
export class PgBossAdapter implements IJobQueue {
private boss: PgBoss;
@@ -35,34 +35,19 @@ export class PgBossAdapter implements IJobQueue {
await this.boss.createQueue(queueName);
}
- async send<T>(queueName: string, data: T): Promise<string> {
- const jobId = await this.boss.send(queueName, data as object);
-
- if (!jobId) {
- throw new Error(`Failed to enqueue job to ${queueName}`);
- }
-
- return jobId;
+ async send<T>(queueName: string, data: T): Promise<void> {
+ await this.boss.send(queueName, data as object);
}
- async work<T>(
- queueName: string,
- config: JobConfig,
- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
- ): Promise<void> {
+ async work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void> {
await this.boss.work(
queueName,
{
- batchSize: config.batchSize ?? 1,
- pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,
+ batchSize: 1,
+ pollingIntervalSeconds: 3,
},
- async (jobs: Job<T>[]) => {
- const normalizedJobs: JobData<T>[] = jobs.map(job => ({
- id: job.id,
- data: job.data,
- }));
-
- await handler(normalizedJobs);
+ async ([job]: Job<T>[]) => {
+ await handler(job.data);
}
);
}
diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.ts
index f47e7ef3e..b1fa693cc 100644
--- a/server/src/queues/jobQueue.ts
+++ b/server/src/queues/jobQueue.ts
@@ -1,49 +1,7 @@
-export interface JobConfig {
- /**
- * Number of jobs to process in a batch (pg-boss specific)
- * For BullMQ, this maps to concurrency
- */
- batchSize?: number;
-
- /**
- * Polling interval in seconds (pg-boss specific)
- * BullMQ uses event-driven model and ignores this
- */
- pollingIntervalSeconds?: number;
-
- /**
- * Number of concurrent workers (BullMQ specific)
- * Falls back to batchSize if not specified
- */
- concurrency?: number;
-
- /**
- * Rate limiting configuration (BullMQ specific)
- */
- limiter?: {
- max: number;
- duration: number;
- };
-}
-
-export interface JobResult {
- success: boolean;
- error?: Error;
-}
-
-export interface JobData<T> {
- id: string;
- data: T;
-}
-
export interface IJobQueue {
start(): Promise<void>;
stop(): Promise<void>;
createQueue(queueName: string): Promise<void>;
- send<T>(queueName: string, data: T): Promise<string>;
- work<T>(
- queueName: string,
- config: JobConfig,
- handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
- ): Promise<void>;
+ send<T>(queueName: string, data: T): Promise<void>;
+ work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;
}
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 1782fb9da..fe1038d34 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -78,8 +78,8 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
export async function registerCsvParseWorker() {
const jobQueue = getJobQueue();
- await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([job]) => {
- const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
+ await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {
+ const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;
let stream: ReturnType<typeof parse> | null = null;
let processingTimeout: NodeJS.Timeout | null = null;
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index f96ea2e0a..407242af4 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -16,8 +16,8 @@ const getImportDataMapping = (platform: string) => {
export async function registerDataInsertWorker() {
const jobQueue = getJobQueue();
- await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([job]) => {
- const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
+ await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {
+ const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job;
// Handle finalization signal
if (allChunksSent) {