Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

use bullmq for cloud

Edward Cui0958945a79f6
diff --git a/server/package-lock.json b/server/package-lock.json
index 5786925d1..5e0ed6b28 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -29,6 +29,7 @@
"drizzle-orm": "0.43.1",
"fastify": "5.3.3",
"fastify-better-auth": "1.1.0",
+ "ioredis": "^5.4.2",
"ip-address": "^10.0.1",
"isbot": "^5.1.28",
"luxon": "3.6.1",
diff --git a/server/package.json b/server/package.json
index c0668572b..6ceebe02c 100644
--- a/server/package.json
+++ b/server/package.json
@@ -41,6 +41,7 @@
"better-auth": "1.3.27",
"bullmq": "^5.56.5",
"dotenv": "16.5.0",
+ "ioredis": "^5.4.2",
"drizzle-orm": "0.43.1",
"fastify": "5.3.3",
"fastify-better-auth": "1.1.0",
diff --git a/server/src/api/sites/importSiteData.ts b/server/src/api/sites/importSiteData.ts
index e1503b2b8..1c4e355ae 100644
--- a/server/src/api/sites/importSiteData.ts
+++ b/server/src/api/sites/importSiteData.ts
@@ -8,7 +8,7 @@ import { DateTime } from "luxon";
import { z } from "zod";
import { IS_CLOUD } from "../../lib/const.js";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import boss from "../../db/postgres/boss.js";
+import { getJobQueue } from "../../queues/jobQueueFactory.js";
import { ImportLimiter } from "../../services/import/importLimiter.js";
import { ImportStatusManager } from "../../services/import/importStatusManager.js";
import { deleteImportFile } from "../../services/import/utils.js";
@@ -141,7 +141,8 @@ export async function importSiteData(
}
try {
- await boss.send(CSV_PARSE_QUEUE, {
+ const jobQueue = getJobQueue();
+ await jobQueue.send(CSV_PARSE_QUEUE, {
site,
importId,
source,
diff --git a/server/src/index.ts b/server/src/index.ts
index 98d1c16a1..9bff26913 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -80,7 +80,7 @@ import { getTrackingConfig } from "./api/sites/getTrackingConfig.js";
import { getSiteImports } from "./api/sites/getSiteImports.js";
import { importSiteData } from "./api/sites/importSiteData.js";
import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";
-import boss from "./db/postgres/boss.js";
+import { getJobQueue } from "./queues/jobQueueFactory.js";
import { registerCsvParseWorker } from "./services/import/workers/csvParseWorker.js";
import { registerDataInsertWorker } from "./services/import/workers/dataInsertWorker.js";
import { createJobQueues } from "./services/import/workers/queues.js";
@@ -434,7 +434,9 @@ const start = async () => {
weeklyReportService.startWeeklyReportCron();
}
- await boss.start();
+ // Initialize and start job queue system
+ const jobQueue = getJobQueue();
+ await jobQueue.start();
await createJobQueues();
await registerCsvParseWorker();
await registerDataInsertWorker();
@@ -461,7 +463,8 @@ const start = async () => {
// });
// }
} catch (err) {
- await boss.stop();
+ const jobQueue = getJobQueue();
+ await jobQueue.stop();
server.log.error(err);
process.exit(1);
}
@@ -492,6 +495,11 @@ const shutdown = async (signal: string) => {
await server.close();
server.log.info("Server closed");
+ // Shutdown job queue system
+ const jobQueue = getJobQueue();
+ await jobQueue.stop();
+ server.log.info("Job queue shut down");
+
// Shutdown uptime service
// await uptimeService.shutdown();
// server.log.info("Uptime service shut down");
diff --git a/server/src/queues/adapters/bullMQAdapter.ts b/server/src/queues/adapters/bullMQAdapter.ts
new file mode 100644
index 000000000..3a9b3ec21
--- /dev/null
+++ b/server/src/queues/adapters/bullMQAdapter.ts
@@ -0,0 +1,144 @@
+import { Job, Queue, QueueEvents, Worker } from "bullmq";
+import type { Redis as RedisType } from "ioredis";
+import RedisModule from "ioredis";
+import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";
+
+// Extract the Redis constructor for NodeNext module resolution
+const Redis = RedisModule.default || RedisModule;
+
+export class BullMQAdapter implements IJobQueue {
+ private connection: RedisType;
+ private queues: Map<string, Queue> = new Map();
+ private workers: Map<string, Worker> = new Map();
+ private queueEvents: Map<string, QueueEvents> = new Map();
+
+ constructor() {
+ this.connection = new Redis({
+ host: process.env.REDIS_HOST || "localhost",
+ port: parseInt(process.env.REDIS_PORT || "6379", 10),
+ password: process.env.REDIS_PASSWORD,
+ maxRetriesPerRequest: null,
+ retryStrategy: (times: number) => {
+ return Math.min(times * 50, 2000);
+ },
+ });
+
+ this.connection.on("error", (error: Error) => {
+ console.error("[BullMQ] Redis connection error:", error);
+ });
+
+ this.connection.on("connect", () => {
+ console.info("[BullMQ] Connected to Redis");
+ });
+ }
+
+ async start(): Promise<void> {
+ // Verify Redis connection
+ await this.connection.ping();
+ console.info("[BullMQ] Started successfully");
+ }
+
+ async stop(): Promise<void> {
+ console.info("[BullMQ] Stopping...");
+
+ // Close all workers first
+ await Promise.all(Array.from(this.workers.values()).map(worker => worker.close()));
+ this.workers.clear();
+
+ // Close all queue events
+ await Promise.all(Array.from(this.queueEvents.values()).map(qe => qe.close()));
+ this.queueEvents.clear();
+
+ // Close all queues
+ await Promise.all(Array.from(this.queues.values()).map(queue => queue.close()));
+ this.queues.clear();
+
+ // Close Redis connection
+ await this.connection.quit();
+ console.info("[BullMQ] Stopped successfully");
+ }
+
+ async createQueue(queueName: string): Promise<void> {
+ if (!this.queues.has(queueName)) {
+ const queue = new Queue(queueName, {
+ connection: this.connection,
+ defaultJobOptions: {
+ attempts: 1, // No retries, same as pg-boss
+ removeOnComplete: true,
+ removeOnFail: false, // Keep failed jobs for debugging
+ },
+ });
+
+ this.queues.set(queueName, queue);
+
+ // Create QueueEvents for monitoring
+ const queueEvents = new QueueEvents(queueName, {
+ connection: this.connection,
+ });
+
+ this.queueEvents.set(queueName, queueEvents);
+
+ // Log job events
+ queueEvents.on("completed", ({ jobId }) => {
+ console.info(`[BullMQ] Job ${jobId} completed in queue ${queueName}`);
+ });
+
+ queueEvents.on("failed", ({ jobId, failedReason }) => {
+ console.error(`[BullMQ] Job ${jobId} failed in queue ${queueName}:`, failedReason);
+ });
+ }
+ }
+
+ async send<T = any>(queueName: string, data: T, options?: { priority?: number; delay?: number }): Promise<string> {
+ const queue = this.queues.get(queueName);
+ if (!queue) {
+ throw new Error(`Queue ${queueName} not found. Call createQueue first.`);
+ }
+
+ const job = await queue.add(queueName, data, {
+ priority: options?.priority,
+ delay: options?.delay,
+ });
+
+ if (!job.id) {
+ throw new Error(`Failed to enqueue job to ${queueName}`);
+ }
+
+ return job.id;
+ }
+
+ async work<T = any>(
+ queueName: string,
+ config: JobConfig,
+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
+ ): Promise<void> {
+ const worker = new Worker(
+ queueName,
+ async (job: Job<T>) => {
+ const normalizedJob: JobData<T> = {
+ id: job.id!,
+ data: job.data,
+ };
+
+ // BullMQ processes one job at a time per worker
+ // We wrap it in an array to match the interface
+ await handler([normalizedJob]);
+ },
+ {
+ connection: this.connection,
+ concurrency: config.concurrency ?? config.batchSize ?? 1,
+ limiter: config.limiter,
+ }
+ );
+
+ worker.on("error", error => {
+ console.error(`[BullMQ] Worker error in queue ${queueName}:`, error);
+ });
+
+ worker.on("failed", (job, error) => {
+ console.error(`[BullMQ] Job ${job?.id} failed in queue ${queueName}:`, error);
+ });
+
+ this.workers.set(queueName, worker);
+ }
+}
diff --git a/server/src/queues/adapters/pgBossAdapter.ts b/server/src/queues/adapters/pgBossAdapter.ts
new file mode 100644
index 000000000..3b6efdbc8
--- /dev/null
+++ b/server/src/queues/adapters/pgBossAdapter.ts
@@ -0,0 +1,76 @@
+import PgBoss, { Job } from "pg-boss";
+import { IJobQueue, JobConfig, JobData, JobResult } from "../jobQueue.js";
+
+export class PgBossAdapter implements IJobQueue {
+ private boss: PgBoss;
+
+ constructor() {
+ this.boss = new PgBoss({
+ host: process.env.POSTGRES_HOST || "postgres",
+ port: parseInt(process.env.POSTGRES_PORT || "5432", 10),
+ database: process.env.POSTGRES_DB,
+ user: process.env.POSTGRES_USER,
+ password: process.env.POSTGRES_PASSWORD,
+ schema: "pgboss",
+ application_name: "data-import-system",
+ retryLimit: 0,
+ });
+
+ this.boss.on("error", (error) => {
+ console.error("[PgBoss] Error:", error);
+ });
+ }
+
+ async start(): Promise<void> {
+ await this.boss.start();
+ console.info("[PgBoss] Started successfully");
+ }
+
+ async stop(): Promise<void> {
+ await this.boss.stop();
+ console.info("[PgBoss] Stopped successfully");
+ }
+
+ async createQueue(queueName: string): Promise<void> {
+ await this.boss.createQueue(queueName);
+ }
+
+ async send<T = any>(
+ queueName: string,
+ data: T,
+ options?: { priority?: number; delay?: number }
+ ): Promise<string> {
+ const jobId = await this.boss.send(queueName, data as object, {
+ priority: options?.priority,
+ startAfter: options?.delay ? new Date(Date.now() + options.delay) : undefined,
+ });
+
+ if (!jobId) {
+ throw new Error(`Failed to enqueue job to ${queueName}`);
+ }
+
+ return jobId;
+ }
+
+ async work<T = any>(
+ queueName: string,
+ config: JobConfig,
+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
+ ): Promise<void> {
+ await this.boss.work(
+ queueName,
+ {
+ batchSize: config.batchSize ?? 1,
+ pollingIntervalSeconds: config.pollingIntervalSeconds ?? 2,
+ },
+ async (jobs: Job<T>[]) => {
+ const normalizedJobs: JobData<T>[] = jobs.map((job) => ({
+ id: job.id,
+ data: job.data,
+ }));
+
+ await handler(normalizedJobs);
+ }
+ );
+ }
+}
diff --git a/server/src/queues/jobQueue.ts b/server/src/queues/jobQueue.ts
new file mode 100644
index 000000000..d80e0f736
--- /dev/null
+++ b/server/src/queues/jobQueue.ts
@@ -0,0 +1,56 @@
+export interface JobConfig {
+ /**
+ * Number of jobs to process in a batch (pg-boss specific)
+ * For BullMQ, this maps to concurrency
+ */
+ batchSize?: number;
+
+ /**
+ * Polling interval in seconds (pg-boss specific)
+ * BullMQ uses event-driven model and ignores this
+ */
+ pollingIntervalSeconds?: number;
+
+ /**
+ * Number of concurrent workers (BullMQ specific)
+ * Falls back to batchSize if not specified
+ */
+ concurrency?: number;
+
+ /**
+ * Rate limiting configuration (BullMQ specific)
+ */
+ limiter?: {
+ max: number;
+ duration: number;
+ };
+}
+
+export interface JobResult {
+ success: boolean;
+ error?: Error;
+}
+
+export interface JobData<T> {
+ id: string;
+ data: T;
+}
+
+export interface IJobQueue {
+ start(): Promise<void>;
+ stop(): Promise<void>;
+ createQueue(queueName: string): Promise<void>;
+ send<T = any>(
+ queueName: string,
+ data: T,
+ options?: {
+ priority?: number;
+ delay?: number;
+ }
+ ): Promise<string>;
+ work<T = any>(
+ queueName: string,
+ config: JobConfig,
+ handler: (jobs: JobData<T>[]) => Promise<void | JobResult>
+ ): Promise<void>;
+}
diff --git a/server/src/queues/jobQueueFactory.ts b/server/src/queues/jobQueueFactory.ts
new file mode 100644
index 000000000..128f8df05
--- /dev/null
+++ b/server/src/queues/jobQueueFactory.ts
@@ -0,0 +1,23 @@
+import { IS_CLOUD } from "../lib/const.js";
+import { IJobQueue } from "./jobQueue.js";
+import { PgBossAdapter } from "./adapters/pgBossAdapter.js";
+import { BullMQAdapter } from "./adapters/bullMQAdapter.js";
+
+let queueInstance: IJobQueue | null = null;
+
+/**
+ * Get the singleton job queue instance
+ * Returns BullMQ for cloud deployments, pg-boss for self-hosted
+ */
+export const getJobQueue = (): IJobQueue => {
+ if (!queueInstance) {
+ if (IS_CLOUD) {
+ console.info("[JobQueue] Initializing BullMQ for cloud deployment");
+ queueInstance = new BullMQAdapter();
+ } else {
+ console.info("[JobQueue] Initializing pg-boss for self-hosted deployment");
+ queueInstance = new PgBossAdapter();
+ }
+ }
+ return queueInstance;
+};
diff --git a/server/src/services/import/utils.ts b/server/src/services/import/utils.ts
index 9dbb12730..adf6af32a 100644
--- a/server/src/services/import/utils.ts
+++ b/server/src/services/import/utils.ts
@@ -1,17 +1,33 @@
import { unlink } from "node:fs/promises";
import { r2Storage } from "../storage/r2StorageService.js";
-export const deleteImportFile = async (storageLocation: string, isR2Storage: boolean) => {
+export interface DeleteFileResult {
+ success: boolean;
+ error?: string;
+}
+
+/**
+ * Delete an import file from storage.
+ * Returns result instead of throwing to prevent worker crashes.
+ */
+export const deleteImportFile = async (
+ storageLocation: string,
+ isR2Storage: boolean
+): Promise<DeleteFileResult> => {
try {
if (isR2Storage) {
await r2Storage.deleteImportFile(storageLocation);
- console.log(`Deleted R2 import file: ${storageLocation}`);
+ console.log(`[Import] Deleted R2 file: ${storageLocation}`);
} else {
await unlink(storageLocation);
- console.log(`Deleted local import file: ${storageLocation}`);
+ console.log(`[Import] Deleted local file: ${storageLocation}`);
}
+ return { success: true };
} catch (error) {
- console.error(`Failed to delete import file ${storageLocation}:`, error);
- throw error;
+ const errorMsg = error instanceof Error ? error.message : "Unknown error";
+ console.error(`[Import] Failed to delete file ${storageLocation}:`, errorMsg);
+
+ // DON'T throw - return error info instead to prevent worker crashes
+ return { success: false, error: errorMsg };
}
-}
+};
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 3c1ea915f..5819d6aab 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -2,8 +2,7 @@ import { access, constants } from "node:fs/promises";
import { createReadStream } from "node:fs";
import { parse } from "@fast-csv/parse";
import { DateTime } from "luxon";
-import { Job } from "pg-boss";
-import boss from "../../../db/postgres/boss.js";
+import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { r2Storage } from "../../storage/r2StorageService.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { UmamiEvent, umamiHeaders } from "../mappings/umami.js";
@@ -40,44 +39,71 @@ const createLocalFileStream = async (storageLocation: string, source: string) =>
}));
};
-const isDateInRange = (dateStr: string, startDate?: string, endDate?: string) => {
- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
- if (!createdAt.isValid) {
- return false;
+/**
+ * Create a date range filter function.
+ * Parses start/end dates once for performance.
+ */
+const createDateRangeFilter = (startDateStr?: string, endDateStr?: string) => {
+ // Parse dates once, not for every row
+ const startDate = startDateStr
+ ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")
+ : null;
+
+ const endDate = endDateStr
+ ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")
+ : null;
+
+ // Validate parsed dates
+ if (startDate && !startDate.isValid) {
+ throw new Error(`Invalid start date: ${startDateStr}`);
+ }
+ if (endDate && !endDate.isValid) {
+ throw new Error(`Invalid end date: ${endDateStr}`);
}
- if (startDate) {
- const start = DateTime.fromFormat(startDate, "yyyy-MM-dd", { zone: "utc" });
- if (!start.isValid || createdAt < start.startOf("day")) {
+ // Return fast filter function
+ return (dateStr: string): boolean => {
+ const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ if (!createdAt.isValid) {
return false;
}
- }
- if (endDate) {
- const end = DateTime.fromFormat(endDate, "yyyy-MM-dd", { zone: "utc" });
- if (!end.isValid || createdAt > end.endOf("day")) {
+ if (startDate && createdAt < startDate) {
return false;
}
- }
- return true;
+ if (endDate && createdAt > endDate) {
+ return false;
+ }
+
+ return true;
+ };
};
export async function registerCsvParseWorker() {
- await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
- const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
+ const jobQueue = getJobQueue();
+
+ await jobQueue.work<CsvParseJob>(
+ CSV_PARSE_QUEUE,
+ { batchSize: 1, pollingIntervalSeconds: 10 },
+ async ([job]) => {
+ const { site, importId, source, storageLocation, isR2Storage, organization, startDate, endDate } = job.data;
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
- await deleteImportFile(storageLocation, isR2Storage);
+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
+ if (!deleteResult.success) {
+ console.warn(`[Import ${importId}] File cleanup failed: ${deleteResult.error}`);
+ }
return;
}
const chunkSize = 5000;
let chunk: UmamiEvent[] = [];
let rowsProcessed = 0;
+ let chunksSent = 0; // Track total chunks sent
const stream = isR2Storage
? await createR2FileStream(storageLocation, source)
@@ -85,8 +111,11 @@ export async function registerCsvParseWorker() {
await ImportStatusManager.updateStatus(importId, "processing");
+ // Create date filter once for performance (not per-row)
+ const isDateInRange = createDateRangeFilter(startDate, endDate);
+
for await (const data of stream) {
- if (!data.created_at || !isDateInRange(data.created_at, startDate, endDate)) {
+ if (!data.created_at || !isDateInRange(data.created_at)) {
continue;
}
@@ -98,44 +127,60 @@ export async function registerCsvParseWorker() {
rowsProcessed++;
if (chunk.length >= chunkSize) {
- await boss.send(DATA_INSERT_QUEUE, {
+ await jobQueue.send(DATA_INSERT_QUEUE, {
site,
importId,
source,
chunk,
+ chunkNumber: chunksSent,
allChunksSent: false,
});
+ chunksSent++;
chunk = [];
}
}
+ // Send final chunk if any data remains
if (chunk.length > 0) {
- await boss.send(DATA_INSERT_QUEUE, {
+ await jobQueue.send(DATA_INSERT_QUEUE, {
site,
importId,
source,
chunk,
+ chunkNumber: chunksSent,
allChunksSent: false,
});
+ chunksSent++;
}
- await boss.send(DATA_INSERT_QUEUE, {
+ // Send finalization signal with total chunk count
+ await jobQueue.send(DATA_INSERT_QUEUE, {
site,
importId,
source,
chunk: [],
+ totalChunks: chunksSent,
allChunksSent: true,
});
- } catch (error) {
- console.error("Error in CSV parse worker:", error);
- await ImportStatusManager.updateStatus(
- importId,
- "failed",
- error instanceof Error ? error.message : "Unknown error occurred"
- );
- throw error;
- } finally {
- await deleteImportFile(storageLocation, isR2Storage);
+ } catch (error) {
+ console.error("Error in CSV parse worker:", error);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ error instanceof Error ? error.message : "Unknown error occurred"
+ );
+ throw error;
+ } finally {
+ // Clean up file - don't throw on failure to prevent worker crashes
+ const deleteResult = await deleteImportFile(storageLocation, isR2Storage);
+ if (!deleteResult.success) {
+ console.warn(
+ `[Import ${importId}] File cleanup failed, will remain in storage: ${deleteResult.error}`
+ );
+ // File will be orphaned but import status is already recorded
+ // Could implement a cleanup job here to retry later
+ }
+ }
}
- });
+ );
}
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index cb1342bf4..dcd601c42 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -1,8 +1,7 @@
-import boss from "../../../db/postgres/boss.js";
+import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { UmamiImportMapper } from "../mappings/umami.js";
import { DataInsertJob, DATA_INSERT_QUEUE } from "./jobs.js";
import { clickhouse } from "../../../db/clickhouse/clickhouse.js";
-import { Job } from "pg-boss";
import { ImportStatusManager } from "../importStatusManager.js";
const getImportDataMapping = (source: string) => {
@@ -12,42 +11,71 @@ const getImportDataMapping = (source: string) => {
default:
throw new Error(`Unsupported import source: ${source}`);
}
-}
+};
export async function registerDataInsertWorker() {
- await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {
- const { site, importId, source, chunk, allChunksSent } = job.data;
+ const jobQueue = getJobQueue();
+
+ await jobQueue.work<DataInsertJob>(
+ DATA_INSERT_QUEUE,
+ { batchSize: 1, pollingIntervalSeconds: 2 },
+ async ([job]) => {
+ const { site, importId, source, chunk, chunkNumber, totalChunks, allChunksSent } = job.data;
- if (allChunksSent) {
+ // Handle finalization signal
+ if (allChunksSent) {
+ try {
+ await ImportStatusManager.updateStatus(importId, "completed");
+ console.log(
+ `[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`
+ );
+ return;
+ } catch (error) {
+ console.error(`[Import ${importId}] Failed to mark as completed:`, error);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ "Failed to complete import"
+ );
+ throw error;
+ }
+ }
+
+ // Process data chunk
try {
- await ImportStatusManager.updateStatus(importId, "completed");
- console.log(`Import ${importId} completed`);
- return;
+ const dataMapper = getImportDataMapping(source);
+ const transformedRecords = dataMapper.transform(chunk, site, importId);
+
+ // Insert to ClickHouse (critical - must succeed)
+ await clickhouse.insert({
+ table: "events",
+ values: transformedRecords,
+ format: "JSONEachRow",
+ });
+
+ // Update progress (non-critical - log if fails but don't crash)
+ try {
+ await ImportStatusManager.updateProgress(importId, transformedRecords.length);
+ } catch (progressError) {
+ console.warn(
+ `[Import ${importId}] Progress update failed (data inserted successfully):`,
+ progressError instanceof Error ? progressError.message : progressError
+ );
+ // Don't throw - data is safely in ClickHouse, progress can be off slightly
+ }
+
+ console.log(
+ `[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`
+ );
} catch (error) {
- console.error(`Failed to mark import ${importId} as completed:`, error);
- await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
+ console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ `Data insertion failed: ${error instanceof Error ? error.message : "Unknown error"}`
+ );
throw error;
}
}
-
- try {
- const dataMapper = getImportDataMapping(source);
- const transformedRecords = dataMapper.transform(chunk, site, importId);
-
- await clickhouse.insert({
- table: "events",
- values: transformedRecords,
- format: "JSONEachRow",
- });
-
- await ImportStatusManager.updateProgress(
- importId,
- transformedRecords.length
- );
- } catch (error) {
- console.error("Error in data insert worker:", error);
- await ImportStatusManager.updateStatus(importId, "failed", "Error inserting chunk");
- throw error;
- }
- });
+ );
}
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 1f4f31a13..0bade893d 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -20,5 +20,7 @@ export interface CsvParseJob extends ImportJob {
export interface DataInsertJob extends ImportJob {
chunk: UmamiEvent[];
- allChunksSent: boolean;
+ chunkNumber?: number; // Chunk sequence number for tracking
+ totalChunks?: number; // Expected total chunks for finalization validation
+ allChunksSent: boolean; // Finalization signal
}
diff --git a/server/src/services/import/workers/queues.ts b/server/src/services/import/workers/queues.ts
index 53d883804..a574e1fd5 100644
--- a/server/src/services/import/workers/queues.ts
+++ b/server/src/services/import/workers/queues.ts
@@ -1,10 +1,12 @@
-import boss from "../../../db/postgres/boss.js";
+import { getJobQueue } from "../../../queues/jobQueueFactory.js";
import { CSV_PARSE_QUEUE, DATA_INSERT_QUEUE } from "./jobs.js";
export const createJobQueues = async () => {
+ const jobQueue = getJobQueue();
+
try {
- await boss.createQueue(CSV_PARSE_QUEUE);
- await boss.createQueue(DATA_INSERT_QUEUE);
+ await jobQueue.createQueue(CSV_PARSE_QUEUE);
+ await jobQueue.createQueue(DATA_INSERT_QUEUE);
} catch (error) {
throw new Error(`Failed to create job queues: ${error instanceof Error ? error.message : "Unknown error"}`);
}