🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor queue initialization and better error handling
Edward Cui256e7b933ee4
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex 2f4541509..c555a20a2 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -61,6 +61,11 @@ type ImportDataRequest = {export async function importSiteData(request: FastifyRequest<ImportDataRequest>, reply: FastifyReply) {try {+ const jobQueue = getJobQueue();+ if (!jobQueue.isReady()) {+ return reply.status(503).send({ error: "Import system is temporarily unavailable. Please try again later." });+ }+const parsedParams = importDataRequestSchema.safeParse({params: request.params,});@@ -147,7 +152,6 @@ export async function importSiteData(request: FastifyRequest<ImportDataRequest>,}try {- const jobQueue = getJobQueue();await jobQueue.send<CsvParseJob>(CSV_PARSE_QUEUE, {site,importId,diff --git a/server/src/index.ts b/server/src/index.tsindex 2b10073dc..5a5c9f898 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -84,9 +84,6 @@ import { getSiteImports } from "./api/sites/getSiteImports.js";import { importSiteData } from "./api/sites/importSiteData.js";import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";import { getJobQueue } from "./queues/jobQueueFactory.js";-import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";-import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";-import { createJobQueues } from "./services/import/workers/queues.js";const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);@@ -454,12 +451,15 @@ const start = async () => {weeklyReportService.startWeeklyReportCron();}- // Initialize and start job queue system+ // Initialize and start import job queue systemconst jobQueue = getJobQueue();- await jobQueue.start();- await createJobQueues();- await registerCsvParseWorker();- await registerDataInsertWorker();+ try {+ await jobQueue.start();+ } catch (queueError) {+ server.log.error("Failed to initialize import job queue system:", queueError);+ server.log.warn("Server will continue without import functionality");+ // Don't exit - allow server to run without import features+ }// Initialize import cleanup service (runs daily to clean up old import files)if (IS_CLOUD) {diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.tsindex 24c9bbe88..0372ba0e3 100644--- a/server/src/queues/adapters/bullMQAdapter.ts+++ b/server/src/queues/adapters/bullMQAdapter.ts@@ -1,11 +1,15 @@import { Job, Queue, QueueEvents, Worker } from "bullmq";import { IJobQueue } from "../jobQueue.js";+import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../services/import/workers/jobs.js";+import { createCsvParseWorker } from "../../services/import/workers/csvParseWorker.js";+import { createDataInsertWorker } from "../../services/import/workers/dataInsertWorker.js";export class BullMQAdapter implements IJobQueue {private readonly connection: { host: string; port: number; password?: string };private queues: Map<string, Queue> = new Map();private workers: Map<string, Worker> = new Map();private queueEvents: Map<string, QueueEvents> = new Map();+ private ready: boolean = false;constructor() {this.connection = {@@ -13,12 +17,23 @@ export class BullMQAdapter implements IJobQueue {port: parseInt(process.env.REDIS_PORT || "6379", 10),...(process.env.REDIS_PASSWORD && { password: process.env.REDIS_PASSWORD }),};-- console.info(`[BullMQ] Configuration: Redis at ${this.connection.host}:${this.connection.port}`);}async start(): Promise<void> {- console.info("[BullMQ] Started successfully");+ try {+ await this.createQueue(CSV_PARSE_QUEUE);+ await this.createQueue(DATA_INSERT_QUEUE);++ await createCsvParseWorker(this);+ await createDataInsertWorker(this);++ this.ready = true;+ console.info("[BullMQ] Started successfully");+ } catch (error) {+ this.ready = false;+ console.error("[BullMQ] Failed to initialize:", error);+ throw error;+ }}async stop(): Promise<void> {@@ -174,4 +189,8 @@ export class BullMQAdapter implements IJobQueue {throw error;}}++ isReady(): boolean {+ return this.ready;+ }}diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.tsindex 780c445c3..125493eb4 100644--- a/server/src/queues/adapters/pgBossAdapter.ts+++ b/server/src/queues/adapters/pgBossAdapter.ts@@ -1,8 +1,12 @@import PgBoss, { Job } from "pg-boss";import { IJobQueue } from "../jobQueue.js";+import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "../../services/import/workers/jobs.js";+import { createCsvParseWorker } from "../../services/import/workers/csvParseWorker.js";+import { createDataInsertWorker } from "../../services/import/workers/dataInsertWorker.js";export class PgBossAdapter implements IJobQueue {private boss: PgBoss;+ private ready: boolean = false;constructor() {this.boss = new PgBoss({@@ -22,8 +26,22 @@ export class PgBossAdapter implements IJobQueue {}async start(): Promise<void> {- await this.boss.start();- console.info("[PgBoss] Started successfully");+ try {+ await this.boss.start();++ await this.createQueue(CSV_PARSE_QUEUE);+ await this.createQueue(DATA_INSERT_QUEUE);++ await createCsvParseWorker(this);+ await createDataInsertWorker(this);++ this.ready = true;+ console.info("[PgBoss] Started successfully");+ } catch (error) {+ this.ready = false;+ console.error("[PgBoss] Failed to initialize:", error);+ throw error;+ }}async stop(): Promise<void> {@@ -51,4 +69,8 @@ export class PgBossAdapter implements IJobQueue {});}++ isReady(): boolean {+ return this.ready;+ }}diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.tsindex b1fa693cc..e83ee669c 100644--- a/server/src/queues/jobQueue.ts+++ b/server/src/queues/jobQueue.ts@@ -4,4 +4,5 @@ export interface IJobQueue {createQueue(queueName: string): Promise<void>;send<T>(queueName: string, data: T): Promise<void>;work<T>(queueName: string, handler: (job: T) => Promise<void>): Promise<void>;+ isReady(): boolean;}diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 8953eb07f..301ab99da 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -2,7 +2,7 @@ import { access, constants } from "node:fs/promises";import { createReadStream } from "node:fs";import { parse } from "@fast-csv/parse";import { DateTime } from "luxon";-import { getJobQueue } from "../../../queues/jobQueueFactory.js";+import { IJobQueue } from "../../../queues/jobQueue.js";import { r2Storage } from "../../storage/r2StorageService.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE, DataInsertJob } from "./jobs.js";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";@@ -75,9 +75,7 @@ const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {};};-export async function registerCsvParseWorker() {- const jobQueue = getJobQueue();-+export async function createCsvParseWorker(jobQueue: IJobQueue) {await jobQueue.work<CsvParseJob>(CSV_PARSE_QUEUE, async job => {const { site, importId, platform, storageLocation, isR2Storage, organization, startDate, endDate } = job;diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 0ee20dbe7..f045d4ea8 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -1,4 +1,4 @@-import { getJobQueue } from "../../../queues/jobQueueFactory.js";+import { IJobQueue } from "../../../queues/jobQueue.js";import { UmamiImportMapper } from "../mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";@@ -13,9 +13,7 @@ const getImportDataMapping = (platform: string) => {}};-export async function registerDataInsertWorker() {- const jobQueue = getJobQueue();-+export async function createDataInsertWorker(jobQueue: IJobQueue) {await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {const { site, importId, platform, chunk, allChunksSent } = job;diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.tsdeleted file mode 100644index b6f60d25f..000000000--- a/server/src/services/import/workers/queues.ts+++ /dev/null@@ -1,13 +0,0 @@-import { getJobQueue } from "../../../queues/jobQueueFactory.js";-import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "./jobs.js";--export const createJobQueues = async () => {- const jobQueue = getJobQueue();-- try {- await jobQueue.createQueue(CSV_PARSE_QUEUE);- await jobQueue.createQueue(DATA_INSERT_QUEUE);- } catch (error) {- throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);- }-};