Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor queue initialization and better error handling

Edward Cui256e7b933ee4
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index 2f4541509..c555a20a2 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -61,6 +61,11 @@ type ImportDataRequest = {
export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {
try {
+ const jobQueue = getJobQueue();
+ if (!jobQueue.isReady()) {
+ return reply.status(503).send({ error: "Import system is temporarily unavailable. Please try again later." });
+ }
+
const parsedParams = importDataRequestSchema.safeParse({
params: request.params,
});
@@ -147,7 +152,6 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,
}
try {
- const jobQueue = getJobQueue();
await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {
site,
importId,
diff --git a/server/src/index.ts b/server/src/index.ts
index 2b10073dc..5a5c9f898 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -84,9 +84,6 @@ import { getSiteImports } from "./api/sites/getSiteImports.js";
import { importSiteData } from "./api/sites/importSiteData.js";
import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";
import { getJobQueue } from "./queues/jobQueueFactory.js";
-import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";
-import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";
-import { createJobQueues } from "./services/import/workers/queues.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -454,12 +451,15 @@ const start = async () => {
weeklyReportService.startWeeklyReportCron();
}
- // Initialize and start job queue system
+ // Initialize and start import job queue system
const jobQueue = getJobQueue();
- await jobQueue.start();
- await createJobQueues();
- await registerCsvParseWorker();
- await registerDataInsertWorker();
+ try {
+ await jobQueue.start();
+ } catch (queueError) {
+ server.log.error("Failed to initialize import job queue system:", queueError);
+ server.log.warn("Server will continue without import functionality");
+ // Don't exit - allow server to run without import features
+ }
// Initialize import cleanup service (runs daily to clean up old import files)
if (IS_CLOUD) {
diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.ts
index 24c9bbe88..0372ba0e3 100644
--- a/server/src/queues/adapters/bullMQAdapter.ts
+++ b/server/src/queues/adapters/bullMQAdapter.ts
@@ -1,11 +1,15 @@
import { Job, Queue, QueueEvents, Worker } from "bullmq";
import { IJobQueue } from "../jobQueue.js";
+import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../services/import/workers/jobs.js";
+import { createCsvParseWorker } from "../../services/import/workers/csvParseWorker.js";
+import { createDataInsertWorker } from "../../services/import/workers/dataInsertWorker.js";
export class BullMQAdapter implements IJobQueue {
private readonly connection: { host: string; port: number; password?: string };
private queues: Map<string, Queue> = new Map();
private workers: Map<string, Worker> = new Map();
private queueEvents: Map<string, QueueEvents> = new Map();
+ private ready: boolean = false;
constructor() {
this.connection = {
@@ -13,12 +17,23 @@ export class BullMQAdapter implements IJobQueue {
port: parseInt(process.env.REDIS_PORT || "6379", 10),
...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD }),
};
-
- console.info(`[BullMQ] Configuration: Redis at ${this.connection.host}:${this.connection.port}`);
}
async start(): Promise<void> {
- console.info("[BullMQ] Started successfully");
+ try {
+ await this.createQueue(CSV_PARSE_QUEUE);
+ await this.createQueue(DATA_INSERT_QUEUE);
+
+ await createCsvParseWorker(this);
+ await createDataInsertWorker(this);
+
+ this.ready = true;
+ console.info("[BullMQ] Started successfully");
+ } catch (error) {
+ this.ready = false;
+ console.error("[BullMQ] Failed to initialize:", error);
+ throw error;
+ }
}
async stop(): Promise<void> {
@@ -174,4 +189,8 @@ export class BullMQAdapter implements IJobQueue {
throw error;
}
}
+
+ isReady(): boolean {
+ return this.ready;
+ }
}
diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.ts
index 780c445c3..125493eb4 100644
--- a/server/src/queues/adapters/pgBossAdapter.ts
+++ b/server/src/queues/adapters/pgBossAdapter.ts
@@ -1,8 +1,12 @@
import PgBoss, { Job } from "pg-boss";
import { IJobQueue } from "../jobQueue.js";
+import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../services/import/workers/jobs.js";
+import { createCsvParseWorker } from "../../services/import/workers/csvParseWorker.js";
+import { createDataInsertWorker } from "../../services/import/workers/dataInsertWorker.js";
export class PgBossAdapter implements IJobQueue {
private boss: PgBoss;
+ private ready: boolean = false;
constructor() {
this.boss = new PgBoss({
@@ -22,8 +26,22 @@ export class PgBossAdapter implements IJobQueue {
}
async start(): Promise<void> {
- await this.boss.start();
- console.info("[PgBoss] Started successfully");
+ try {
+ await this.boss.start();
+
+ await this.createQueue(CSV_PARSE_QUEUE);
+ await this.createQueue(DATA_INSERT_QUEUE);
+
+ await createCsvParseWorker(this);
+ await createDataInsertWorker(this);
+
+ this.ready = true;
+ console.info("[PgBoss] Started successfully");
+ } catch (error) {
+ this.ready = false;
+ console.error("[PgBoss] Failed to initialize:", error);
+ throw error;
+ }
}
async stop(): Promise<void> {
@@ -51,4 +69,8 @@ export class PgBossAdapter implements IJobQueue {
}
);
}
+
+ isReady(): boolean {
+ return this.ready;
+ }
}
diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.ts
index b1fa693cc..e83ee669c 100644
--- a/server/src/queues/jobQueue.ts
+++ b/server/src/queues/jobQueue.ts
@@ -4,4 +4,5 @@ export interface IJobQueue {
createQueue(queueName: string): Promise<void>;
send<T>(queueName: string, data: T): Promise<void>;
work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;
+ isReady(): boolean;
}
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 8953eb07f..301ab99da 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -2,7 +2,7 @@ import { access, constants } from "node:fs/promises";
import { createReadStream } from "node:fs";
import { parse } from "@fast-csv/parse";
import { DateTime } from "luxon";
-import { getJobQueue } from "../../../queues/jobQueueFactory.js";
+import { IJobQueue } from "../../../queues/jobQueue.js";
import { r2Storage } from "../../storage/r2StorageService.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
@@ -75,9 +75,7 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
};
};
-export async function registerCsvParseWorker() {
- const jobQueue = getJobQueue();
-
+export async function createCsvParseWorker(jobQueue: IJobQueue) {
await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {
const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 0ee20dbe7..f045d4ea8 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -1,4 +1,4 @@
-import { getJobQueue } from "../../../queues/jobQueueFactory.js";
+import { IJobQueue } from "../../../queues/jobQueue.js";
import { UmamiImportMapper } from "../mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
@@ -13,9 +13,7 @@ const getImportDataMapping = (platform: string) => {
}
};
-export async function registerDataInsertWorker() {
- const jobQueue = getJobQueue();
-
+export async function createDataInsertWorker(jobQueue: IJobQueue) {
await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {
const { site, importId, platform, chunk, allChunksSent } = job;
diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.ts
deleted file mode 100644
index b6f60d25f..000000000
--- a/server/src/services/import/workers/queues.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { getJobQueue } from "../../../queues/jobQueueFactory.js";
-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "./jobs.js";
-
-export const createJobQueues = async () => {
- const jobQueue = getJobQueue();
-
- try {
- await jobQueue.createQueue(CSV_PARSE_QUEUE);
- await jobQueue.createQueue(DATA_INSERT_QUEUE);
- } catch (error) {
- throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);
- }
-};