🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
use bullmq for cloud
Edward Cui0958945a79f6
diff --git a/server/package-lock.json b/server/package-lock.jsonindex 5786925d1..5e0ed6b28 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -29,6 +29,7 @@"drizzle-orm": "0.43.1","fastify": "5.3.3","fastify-better-auth": "1.1.0",+ "ioredis": "^5.4.2","ip-address": "^10.0.1","isbot": "^5.1.28","luxon": "3.6.1",diff --git a/server/package.json b/server/package.jsonindex c0668572b..6ceebe02c 100644--- a/server/package.json+++ b/server/package.json@@ -41,6 +41,7 @@"better-auth": "1.3.27","bullmq": "^5.56.5","dotenv": "16.5.0",+ "ioredis": "^5.4.2","drizzle-orm": "0.43.1","fastify": "5.3.3","fastify-better-auth": "1.1.0",diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.tsindex e1503b2b8..1c4e355ae 100644--- a/server/src/api/sites/importSiteData.ts+++ b/server/src/api/sites/importSiteData.ts@@ -8,7 +8,7 @@ import { DateTime } from "luxon";import { z } from "zod";import { IS_CLOUD } from "../../lib/const.js";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import boss from "../../db/postgres/boss.js";+import { getJobQueue } from "../../queues/jobQueueFactory.js";import { ImportLimiter } from "../../services/import/importLimiter.js";import { ImportStatusManager } from "../../services/import/importStatusManager.js";import { deleteImportFile } from "../../services/import/utils.js";@@ -141,7 +141,8 @@ export async function importSiteData(}try {- await boss.send(CSV_PARSE_QUEUE, {+ const jobQueue = getJobQueue();+ await jobQueue.send(CSV_PARSE_QUEUE, {site,importId,source,diff --git a/server/src/index.ts b/server/src/index.tsindex 98d1c16a1..9bff26913 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -80,7 +80,7 @@ import { getTrackingConfig } from "./api/sites/getTrackingConfig.js";import { getSiteImports } from "./api/sites/getSiteImports.js";import { importSiteData } from "./api/sites/importSiteData.js";import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";-import boss from "./db/postgres/boss.js";+import { getJobQueue } from "./queues/jobQueueFactory.js";import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";import { createJobQueues } from "./services/import/workers/queues.js";@@ -434,7 +434,9 @@ const start = async () => {weeklyReportService.startWeeklyReportCron();}- await boss.start();+ // Initialize and start job queue system+ const jobQueue = getJobQueue();+ await jobQueue.start();await createJobQueues();await registerCsvParseWorker();await registerDataInsertWorker();@@ -461,7 +463,8 @@ const start = async () => {// });// }} catch (err) {- await boss.stop();+ const jobQueue = getJobQueue();+ await jobQueue.stop();server.log.error(err);process.exit(1);}@@ -492,6 +495,11 @@ const shutdown = async (signal: string) => {await server.close();server.log.info("Server closed");+ // Shutdown job queue system+ const jobQueue = getJobQueue();+ await jobQueue.stop();+ server.log.info("Job queue shut down");+// Shutdown uptime service// await uptimeService.shutdown();// server.log.info("Uptime service shut down");diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.tsnew file mode 100644index 000000000..3a9b3ec21--- /dev/null+++ b/server/src/queues/adapters/bullMQAdapter.ts@@ -0,0 +1,144 @@+import { Job, Queue, QueueEvents, Worker } from "bullmq";+import type { Redis as RedisType } from "ioredis";+import RedisModule from "ioredis";+import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";++// Extract the Redis constructor for NodeNext module resolution+const Redis = RedisModule.default || RedisModule;++export class BullMQAdapter implements IJobQueue {+ private connection: RedisType;+ private queues: Map<string, Queue> = new Map();+ private workers: Map<string, Worker> = new Map();+ private queueEvents: Map<string, QueueEvents> = new Map();++ constructor() {+ this.connection = new Redis({+ host: process.env.REDIS_HOST || "localhost",+ port: parseInt(process.env.REDIS_PORT || "6379", 10),+ password: process.env.REDIS_PASSWORD,+ maxRetriesPerRequest: null,+ retryStrategy: (times: number) => {+ return Math.min(times * 50, 2000);+ },+ });++ this.connection.on("error", (error: Error) => {+ console.error("[BullMQ] Redis connection error:", error);+ });++ this.connection.on("connect", () => {+ console.info("[BullMQ] Connected to Redis");+ });+ }++ async start(): Promise<void> {+ // Verify Redis connection+ await this.connection.ping();+ console.info("[BullMQ] Started successfully");+ }++ async stop(): Promise<void> {+ console.info("[BullMQ] Stopping...");++ // Close all workers first+ await Promise.all(Array.from(this.workers.values()).map(worker => worker.close()));+ this.workers.clear();++ // Close all queue events+ await Promise.all(Array.from(this.queueEvents.values()).map(qe => qe.close()));+ this.queueEvents.clear();++ // Close all queues+ await Promise.all(Array.from(this.queues.values()).map(queue => queue.close()));+ this.queues.clear();++ // Close Redis connection+ await this.connection.quit();+ console.info("[BullMQ] Stopped successfully");+ }++ async createQueue(queueName: string): Promise<void> {+ if (!this.queues.has(queueName)) {+ const queue = new Queue(queueName, {+ connection: this.connection,+ defaultJobOptions: {+ attempts: 1, // No retries, same as pg-boss+ removeOnComplete: true,+ removeOnFail: false, // Keep failed jobs for debugging+ },+ });++ this.queues.set(queueName, queue);++ // Create QueueEvents for monitoring+ const queueEvents = new QueueEvents(queueName, {+ connection: this.connection,+ });++ this.queueEvents.set(queueName, queueEvents);++ // Log job events+ queueEvents.on("completed", ({ jobId }) => {+ console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);+ });++ queueEvents.on("failed", ({ jobId, failedReason }) => {+ console.error(`[BullMQ] Job ${jobId} failed in queue ${queueName}:`, failedReason);+ });+ }+ }++ async send<T = any>(queueName: string, data: T, options?: { priority?: number; delay?: number }): Promise<string> {+ const queue = this.queues.get(queueName);+ if (!queue) {+ throw new Error(`Queue ${queueName} not found. Call createQueue first.`);+ }++ const job = await queue.add(queueName, data, {+ priority: options?.priority,+ delay: options?.delay,+ });++ if (!job.id) {+ throw new Error(`Failed to enqueue job to ${queueName}`);+ }++ return job.id;+ }++ async work<T = any>(+ queueName: string,+ config: JobConfig,+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>+ ): Promise<void> {+ const worker = new Worker(+ queueName,+ async (job: Job<T>) => {+ const normalizedJob: JobData<T> = {+ id: job.id!,+ data: job.data,+ };++ // BullMQ processes one job at a time per worker+ // We wrap it in an array to match the interface+ await handler([normalizedJob]);+ },+ {+ connection: this.connection,+ concurrency: config.concurrency ?? config.batchSize ?? 1,+ limiter: config.limiter,+ }+ );++ worker.on("error", error => {+ console.error(`[BullMQ] Worker error in queue ${queueName}:`, error);+ });++ worker.on("failed", (job, error) => {+ console.error(`[BullMQ] Job ${job?.id} failed in queue ${queueName}:`, error);+ });++ this.workers.set(queueName, worker);+ }+}diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.tsnew file mode 100644index 000000000..3b6efdbc8--- /dev/null+++ b/server/src/queues/adapters/pgBossAdapter.ts@@ -0,0 +1,76 @@+import PgBoss, { Job } from "pg-boss";+import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";++export class PgBossAdapter implements IJobQueue {+ private boss: PgBoss;++ constructor() {+ this.boss = new PgBoss({+ host: process.env.POSTGRES_HOST || "postgres",+ port: parseInt(process.env.POSTGRES_PORT || "5432", 10),+ database: process.env.POSTGRES_DB,+ user: process.env.POSTGRES_USER,+ password: process.env.POSTGRES_PASSWORD,+ schema: "pgboss",+ application_name: "data-import-system",+ retryLimit: 0,+ });++ this.boss.on("error", (error) => {+ console.error("[PgBoss] Error:", error);+ });+ }++ async start(): Promise<void> {+ await this.boss.start();+ console.info("[PgBoss] Started successfully");+ }++ async stop(): Promise<void> {+ await this.boss.stop();+ console.info("[PgBoss] Stopped successfully");+ }++ async createQueue(queueName: string): Promise<void> {+ await this.boss.createQueue(queueName);+ }++ async send<T = any>(+ queueName: string,+ data: T,+ options?: { priority?: number; delay?: number }+ ): Promise<string> {+ const jobId = await this.boss.send(queueName, data as object, {+ priority: options?.priority,+ startAfter: options?.delay ? new Date(Date.now() + options.delay) : undefined,+ });++ if (!jobId) {+ throw new Error(`Failed to enqueue job to ${queueName}`);+ }++ return jobId;+ }++ async work<T = any>(+ queueName: string,+ config: JobConfig,+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>+ ): Promise<void> {+ await this.boss.work(+ queueName,+ {+ batchSize: config.batchSize ?? 1,+ pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,+ },+ async (jobs: Job<T>[]) => {+ const normalizedJobs: JobData<T>[] = jobs.map((job) => ({+ id: job.id,+ data: job.data,+ }));++ await handler(normalizedJobs);+ }+ );+ }+}diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.tsnew file mode 100644index 000000000..d80e0f736--- /dev/null+++ b/server/src/queues/jobQueue.ts@@ -0,0 +1,56 @@+export interface JobConfig {+ /**+ * Number of jobs to process in a batch (pg-boss specific)+ * For BullMQ, this maps to concurrency+ */+ batchSize?: number;++ /**+ * Polling interval in seconds (pg-boss specific)+ * BullMQ uses event-driven model and ignores this+ */+ pollingIntervalSeconds?: number;++ /**+ * Number of concurrent workers (BullMQ specific)+ * Falls back to batchSize if not specified+ */+ concurrency?: number;++ /**+ * Rate limiting configuration (BullMQ specific)+ */+ limiter?: {+ max: number;+ duration: number;+ };+}++export interface JobResult {+ success: boolean;+ error?: Error;+}++export interface JobData<T> {+ id: string;+ data: T;+}++export interface IJobQueue {+ start(): Promise<void>;+ stop(): Promise<void>;+ createQueue(queueName: string): Promise<void>;+ send<T = any>(+ queueName: string,+ data: T,+ options?: {+ priority?: number;+ delay?: number;+ }+ ): Promise<string>;+ work<T = any>(+ queueName: string,+ config: JobConfig,+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>+ ): Promise<void>;+}diff --git a/server/src/queues/jobQueueFactory.ts b/server/src/queues/jobQueueFactory.tsnew file mode 100644index 000000000..128f8df05--- /dev/null+++ b/server/src/queues/jobQueueFactory.ts@@ -0,0 +1,23 @@+import { IS_CLOUD } from "../lib/const.js";+import { IJobQueue } from "./jobQueue.js";+import { PgBossAdapter } from "./adapters/pgBossAdapter.js";+import { BullMQAdapter } from "./adapters/bullMQAdapter.js";++let queueInstance: IJobQueue | null = null;++/**+ * Get the singleton job queue instance+ * Returns BullMQ for cloud deployments, pg-boss for self-hosted+ */+export const getJobQueue = (): IJobQueue => {+ if (!queueInstance) {+ if (IS_CLOUD) {+ console.info("[JobQueue] Initializing BullMQ for cloud deployment");+ queueInstance = new BullMQAdapter();+ } else {+ console.info("[JobQueue] Initializing pg-boss for self-hosted deployment");+ queueInstance = new PgBossAdapter();+ }+ }+ return queueInstance;+};diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.tsindex 9dbb12730..adf6af32a 100644--- a/server/src/services/import/utils.ts+++ b/server/src/services/import/utils.ts@@ -1,17 +1,33 @@import { unlink } from "node:fs/promises";import { r2Storage } from "../storage/r2StorageService.js";-export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean) => {+export interface DeleteFileResult {+ success: boolean;+ error?: string;+}++/**+ * Delete an import file from storage.+ * Returns result instead of throwing to prevent worker crashes.+ */+export const deleteImportFile = async (+ storageLocation: string,+ isR2Storage: boolean+): Promise<DeleteFileResult> => {try {if (isR2Storage) {await r2Storage.deleteImportFile(storageLocation);- console.log(`Deleted R2 import file: ${storageLocation}`);+ console.log(`[Import] Deleted R2 file: ${storageLocation}`);} else {await unlink(storageLocation);- console.log(`Deleted local import file: ${storageLocation}`);+ console.log(`[Import] Deleted local file: ${storageLocation}`);}+ return { success: true };} catch (error) {- console.error(`Failed to delete import file ${storageLocation}:`, error);- throw error;+ const errorMsg = error instanceof Error ? error.message : "Unknown error";+ console.error(`[Import] Failed to delete file ${storageLocation}:`, errorMsg);++ // DON'T throw - return error info instead to prevent worker crashes+ return { success: false, error: errorMsg };}-}+};diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 3c1ea915f..5819d6aab 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -2,8 +2,7 @@ import { access, constants } from "node:fs/promises";import { createReadStream } from "node:fs";import { parse } from "@fast-csv/parse";import { DateTime } from "luxon";-import { Job } from "pg-boss";-import boss from "../../../db/postgres/boss.js";+import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { r2Storage } from "../../storage/r2StorageService.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";@@ -40,44 +39,71 @@ const createLocalFileStream = async (storageLocation: string, source: string) =>}));};-const isDateInRange = (dateStr: string, startDate?: string, endDate?: string) => {- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });- if (!createdAt.isValid) {- return false;+/**+ * Create a date range filter function.+ * Parses start/end dates once for performance.+ */+const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {+ // Parse dates once, not for every row+ const startDate = startDateStr+ ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")+ : null;++ const endDate = endDateStr+ ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")+ : null;++ // Validate parsed dates+ if (startDate && !startDate.isValid) {+ throw new Error(`Invalid start date: ${startDateStr}`);+ }+ if (endDate && !endDate.isValid) {+ throw new Error(`Invalid end date: ${endDateStr}`);}- if (startDate) {- const start = DateTime.fromFormat(startDate, "yyyy-MM-dd", { zone: "utc" });- if (!start.isValid || createdAt < start.startOf("day")) {+ // Return fast filter function+ return (dateStr: string): boolean => {+ const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ if (!createdAt.isValid) {return false;}- }- if (endDate) {- const end = DateTime.fromFormat(endDate, "yyyy-MM-dd", { zone: "utc" });- if (!end.isValid || createdAt > end.endOf("day")) {+ if (startDate && createdAt < startDate) {return false;}- }- return true;+ if (endDate && createdAt > endDate) {+ return false;+ }++ return true;+ };};export async function registerCsvParseWorker() {- await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;+ const jobQueue = getJobQueue();++ await jobQueue.work<CsvParseJob>(+ CSV_PARSE_QUEUE,+ { batchSize: 1, pollingIntervalSeconds: 10 },+ async ([job]) => {+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;try {const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");- await deleteImportFile(storageLocation, isR2Storage);+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);+ if (!deleteResult.success) {+ console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);+ }return;}const chunkSize = 5000;let chunk: UmamiEvent[] = [];let rowsProcessed = 0;+ let chunksSent = 0; // Track total chunks sentconst stream = isR2Storage? await createR2FileStream(storageLocation, source)@@ -85,8 +111,11 @@ export async function registerCsvParseWorker() {await ImportStatusManager.updateStatus(importId, "processing");+ // Create date filter once for performance (not per-row)+ const isDateInRange = createDateRangeFilter(startDate, endDate);+for await (const data of stream) {- if (!data.created_at || !isDateInRange(data.created_at, startDate, endDate)) {+ if (!data.created_at || !isDateInRange(data.created_at)) {continue;}@@ -98,44 +127,60 @@ export async function registerCsvParseWorker() {rowsProcessed++;if (chunk.length >= chunkSize) {- await boss.send(DATA_INSERT_QUEUE, {+ await jobQueue.send(DATA_INSERT_QUEUE, {site,importId,source,chunk,+ chunkNumber: chunksSent,allChunksSent: false,});+ chunksSent++;chunk = [];}}+ // Send final chunk if any data remainsif (chunk.length > 0) {- await boss.send(DATA_INSERT_QUEUE, {+ await jobQueue.send(DATA_INSERT_QUEUE, {site,importId,source,chunk,+ chunkNumber: chunksSent,allChunksSent: false,});+ chunksSent++;}- await boss.send(DATA_INSERT_QUEUE, {+ // Send finalization signal with total chunk count+ await jobQueue.send(DATA_INSERT_QUEUE, {site,importId,source,chunk: [],+ totalChunks: chunksSent,allChunksSent: true,});- } catch (error) {- console.error("Error in CSV parse worker:", error);- await ImportStatusManager.updateStatus(- importId,- "failed",- error instanceof Error ? error.message : "Unknown error occurred"- );- throw error;- } finally {- await deleteImportFile(storageLocation, isR2Storage);+ } catch (error) {+ console.error("Error in CSV parse worker:", error);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ error instanceof Error ? error.message : "Unknown error occurred"+ );+ throw error;+ } finally {+ // Clean up file - don't throw on failure to prevent worker crashes+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);+ if (!deleteResult.success) {+ console.warn(+ `[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`+ );+ // File will be orphaned but import status is already recorded+ // Could implement a cleanup job here to retry later+ }+ }}- });+ );}diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex cb1342bf4..dcd601c42 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -1,8 +1,7 @@-import boss from "../../../db/postgres/boss.js";+import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { UmamiImportMapper } from "../mappings/umami.js";import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";import { clickhouse } from "../../../db/clickhouse/clickhouse.js";-import { Job } from "pg-boss";import { ImportStatusManager } from "../importStatusManager.js";const getImportDataMapping = (source: string) => {@@ -12,42 +11,71 @@ const getImportDataMapping = (source: string) => {default:throw new Error(`Unsupported import source: ${source}`);}-}+};export async function registerDataInsertWorker() {- await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {- const { site, importId, source, chunk, allChunksSent } = job.data;+ const jobQueue = getJobQueue();++ await jobQueue.work<DataInsertJob>(+ DATA_INSERT_QUEUE,+ { batchSize: 1, pollingIntervalSeconds: 2 },+ async ([job]) => {+ const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;- if (allChunksSent) {+ // Handle finalization signal+ if (allChunksSent) {+ try {+ await ImportStatusManager.updateStatus(importId, "completed");+ console.log(+ `[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`+ );+ return;+ } catch (error) {+ console.error(`[Import ${importId}] Failed to mark as completed:`, error);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ "Failed to complete import"+ );+ throw error;+ }+ }++ // Process data chunktry {- await ImportStatusManager.updateStatus(importId, "completed");- console.log(`Import ${importId} completed`);- return;+ const dataMapper = getImportDataMapping(source);+ const transformedRecords = dataMapper.transform(chunk, site, importId);++ // Insert to ClickHouse (critical - must succeed)+ await clickhouse.insert({+ table: "events",+ values: transformedRecords,+ format: "JSONEachRow",+ });++ // Update progress (non-critical - log if fails but don't crash)+ try {+ await ImportStatusManager.updateProgress(importId, transformedRecords.length);+ } catch (progressError) {+ console.warn(+ `[Import ${importId}] Progress update failed (data inserted successfully):`,+ progressError instanceof Error ? progressError.message : progressError+ );+ // Don't throw - data is safely in ClickHouse, progress can be off slightly+ }++ console.log(+ `[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`+ );} catch (error) {- console.error(`Failed to mark import ${importId} as completed:`, error);- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");+ console.error(`[Import ${importId}] ClickHouse insert failed:`, error);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`+ );throw error;}}-- try {- const dataMapper = getImportDataMapping(source);- const transformedRecords = dataMapper.transform(chunk, site, importId);-- await clickhouse.insert({- table: "events",- values: transformedRecords,- format: "JSONEachRow",- });-- await ImportStatusManager.updateProgress(- importId,- transformedRecords.length- );- } catch (error) {- console.error("Error in data insert worker:", error);- await ImportStatusManager.updateStatus(importId, "failed", "Error inserting chunk");- throw error;- }- });+ );}diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex 1f4f31a13..0bade893d 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -20,5 +20,7 @@ export interface CsvParseJob extends ImportJob {export interface DataInsertJob extends ImportJob {chunk: UmamiEvent[];- allChunksSent: boolean;+ chunkNumber?: number; // Chunk sequence number for tracking+ totalChunks?: number; // Expected total chunks for finalization validation+ allChunksSent: boolean; // Finalization signal}diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.tsindex 53d883804..a574e1fd5 100644--- a/server/src/services/import/workers/queues.ts+++ b/server/src/services/import/workers/queues.ts@@ -1,10 +1,12 @@-import boss from "../../../db/postgres/boss.js";+import { getJobQueue } from "../../../queues/jobQueueFactory.js";import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "./jobs.js";export const createJobQueues = async () => {+ const jobQueue = getJobQueue();+try {- await boss.createQueue(CSV_PARSE_QUEUE);- await boss.createQueue(DATA_INSERT_QUEUE);+ await jobQueue.createQueue(CSV_PARSE_QUEUE);+ await jobQueue.createQueue(DATA_INSERT_QUEUE);} catch (error) {throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);}