Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor and register workers

Edward Cui4a4a82a827cd
diff --git a/server/src/api/importData.ts b/server/src/api/importData.ts
index 95ecbef9c..203249969 100644
--- a/server/src/api/importData.ts
+++ b/server/src/api/importData.ts
@@ -6,7 +6,7 @@ import { z } from "zod";
import crypto from "crypto";
import boss from "../lib/boss.js";
import { getUserHasAccessToSite } from "../lib/auth-utils.js";
-import { importInitiationQueue } from "../types/import.js";
+import { CSV_PARSE_QUEUE } from "../types/import.js";
const IMPORT_DIR = "/tmp/imports";
@@ -73,7 +73,7 @@ export async function importData(
}
try {
- await boss.send(importInitiationQueue, {
+ await boss.send(CSV_PARSE_QUEUE, {
tempFilePath,
organization,
site,
diff --git a/server/src/index.ts b/server/src/index.ts
index 616f45610..0ddbbcadb 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -71,6 +71,8 @@ import { trackEvent } from "./tracker/trackEvent.js";
import { extractSiteId, isSitePublic } from "./utils.js";
import { importData } from "./api/importData.js";
import boss from "./lib/boss.js";
+import { registerCsvParseWorker } from "./workers/csvParseWorker.js";
+import { registerDataInsertWorker } from "./workers/dataInsertWorker.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
@@ -337,6 +339,8 @@ const start = async () => {
await siteConfig.loadSiteConfigs();
await boss.start();
+ await registerCsvParseWorker();
+ await registerDataInsertWorker();
// Start the server
await server.listen({ port: 3001, host: "0.0.0.0" });
diff --git a/server/src/types/import.ts b/server/src/types/import.ts
index 1390185d7..cfc73a783 100644
--- a/server/src/types/import.ts
+++ b/server/src/types/import.ts
@@ -1,6 +1,6 @@
-export const importInitiationQueue = "import-initiation";
+export const CSV_PARSE_QUEUE = "csv-parse";
-export const processImportChunkQueue = "process-import-chunk";
+export const DATA_INSERT_QUEUE = "data-insert";
interface ImportJob {
site: string;
@@ -8,12 +8,12 @@ interface ImportJob {
source: "umami";
}
-export interface ImportInitiationJob extends ImportJob {
+export interface CsvParseJob extends ImportJob {
tempFilePath: string;
organization: string;
}
-export interface ProcessImportChunkJob<T> extends ImportJob {
+export interface DataInsertJob<T> extends ImportJob {
chunk: T[];
chunkNumber: number;
}
diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.ts
new file mode 100644
index 000000000..1b072468a
--- /dev/null
+++ b/server/src/workers/csvParseWorker.ts
@@ -0,0 +1,154 @@
+import fs from "fs";
+import { parseStream } from "@fast-csv/parse";
+import boss from "../lib/boss.js";
+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";
+import { Job } from "pg-boss";
+import { umamiHeaders } from "./mappings/umami.js";
+import { clickhouse } from "../db/clickhouse/clickhouse.js";
+import { processResults } from "../api/analytics/utils.js";
+
+export async function registerCsvParseWorker() {
+ await boss.work(CSV_PARSE_QUEUE, async (job: Job<CsvParseJob>[]) => {
+ try {
+ const {site, importId, source, tempFilePath, organization} = job[0].data;
+
+ const query = await clickhouse.query({
+ query: `
+ SELECT count() AS count
+ FROM events
+ WHERE site_id = {siteId:UInt16}
+ AND import_id IS NOT NULL
+ `,
+ format: "JSONEachRow",
+ query_params: {
+ siteId: Number(site),
+ },
+ });
+ const result = await processResults<{ count: number }>(query);
+ const importedRowCount = result[0].count;
+
+ const maxImportedRows = 1000000;
+ const remainingRows = maxImportedRows - importedRowCount;
+ if (remainingRows <= 0) {
+ console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);
+ await fs.promises.unlink(tempFilePath);
+ return;
+ }
+
+ const chunkSize = 1000;
+ const maxChunks = Math.ceil(remainingRows / chunkSize);
+ let chunkNumber = 0;
+ let chunk: any[] = [];
+ let shouldStop = false;
+
+ const stream = fs.createReadStream(tempFilePath);
+
+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {
+ if (arr1.length !== arr2.length) {
+ return false;
+ }
+ return arr1.every((value, index) => value === arr2[index]);
+ }
+
+ const validateHeaders = (headers: string[]): boolean => {
+ switch (source) {
+ case "umami":
+ return arraysAreEqual(umamiHeaders, headers);
+ default:
+ return false;
+ }
+ };
+
+ const cleanup = async (reason: string) => {
+ console.log(`Stopping CSV processing: ${reason}`);
+ shouldStop = true;
+
+ if (!stream.destroyed) {
+ stream.destroy();
+ }
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ };
+
+ parseStream(stream, {headers: true})
+ .on("headers", async (headers) => {
+ if (!validateHeaders(headers)) {
+ await cleanup("Header validation failed");
+ return;
+ }
+ })
+ .on("data", async (row) => {
+ if (shouldStop) {
+ return;
+ }
+
+ chunk.push(row);
+
+ if (chunk.length >= chunkSize) {
+ chunkNumber++;
+
+ if (chunkNumber > maxChunks) {
+ await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);
+ return;
+ }
+
+ try {
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ });
+ } catch (error) {
+ console.error("Failed to send chunk to queue:", error);
+ await cleanup("Error sending chunk to processing queue");
+ }
+
+ chunk = [];
+ }
+ })
+ .on("end", async (rowCount: number) => {
+ try {
+ if (!shouldStop && chunk.length > 0) {
+ chunkNumber++;
+
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ });
+ }
+
+ if (!shouldStop) {
+ console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ }
+ } catch (error) {
+ console.error("Error during final CSV chunk processing:", error);
+ await cleanup("Error during final chunk processing");
+ }
+ })
+ .on("error", async (error) => {
+ console.error("CSV parsing error:", error);
+ await cleanup("CSV parsing error occurred");
+ });
+ } catch (error) {
+ console.error("Error in importInitiationQueue worker:", error);
+ // optional: notify, requeue, or cleanup
+ }
+ });
+}
diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.ts
new file mode 100644
index 000000000..f154e7ad4
--- /dev/null
+++ b/server/src/workers/dataInsertWorker.ts
@@ -0,0 +1,35 @@
+import boss from "../lib/boss.js";
+import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";
+import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";
+import { clickhouse } from "../db/clickhouse/clickhouse.js";
+import { Job } from "pg-boss";
+
+export async function registerDataInsertWorker() {
+ await boss.work(DATA_INSERT_QUEUE, async (job: Job<DataInsertJob<UmamiEvent>>[]) => {
+ try {
+ const {site, importId, source, chunk, chunkNumber} = job[0].data;
+
+ const dataMapper = getImportDataMapping(source);
+ const transformedRecords = dataMapper.transform(chunk, site, importId);
+
+ await clickhouse.insert({
+ table: "events",
+ values: transformedRecords,
+ format: "JSONEachRow",
+ });
+
+ console.log(`Processed chunk ${chunkNumber} for import ${importId}`);
+ } catch (error) {
+ console.error("Error in processImportChunkQueue worker:", error);
+ }
+ });
+
+ function getImportDataMapping(source: string) {
+ switch (source) {
+ case "umami":
+ return new UmamiImportMapper();
+ default:
+ throw new Error(`Unsupported import source: ${source}`);
+ }
+ }
+}
diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.ts
deleted file mode 100644
index c84a9c0e7..000000000
--- a/server/src/workers/importInitiation.ts
+++ /dev/null
@@ -1,152 +0,0 @@
-import fs from "fs";
-import { parseStream } from "@fast-csv/parse";
-import boss from "../lib/boss.js";
-import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";
-import { Job } from "pg-boss";
-import { umamiHeaders } from "./mappings/umami.js";
-import { clickhouse } from "../db/clickhouse/clickhouse.js";
-import { processResults } from "../api/analytics/utils.js";
-
-await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {
- try {
- const { site, importId, source, tempFilePath, organization } = job[0].data;
-
- const query = await clickhouse.query({
- query: `
- SELECT count() AS count
- FROM events
- WHERE site_id = {siteId:UInt16}
- AND import_id IS NOT NULL
- `,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- });
- const result = await processResults<{ count: number }>(query);
- const importedRowCount = result[0].count;
-
- const maxImportedRows = 1000000;
- const remainingRows = maxImportedRows - importedRowCount;
- if (remainingRows <= 0) {
- console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);
- await fs.promises.unlink(tempFilePath);
- return;
- }
-
- const chunkSize = 1000;
- const maxChunks = Math.ceil(remainingRows / chunkSize);
- let chunkNumber = 0;
- let chunk: any[] = [];
- let shouldStop = false;
-
- const stream = fs.createReadStream(tempFilePath);
-
- const arraysAreEqual = (arr1: string[], arr2: string[]) => {
- if (arr1.length !== arr2.length) {
- return false;
- }
- return arr1.every((value, index) => value === arr2[index]);
- }
-
- const validateHeaders = (headers: string[]): boolean => {
- switch (source) {
- case "umami":
- return arraysAreEqual(umamiHeaders, headers);
- default:
- return false;
- }
- };
-
- const cleanup = async (reason: string) => {
- console.log(`Stopping CSV processing: ${reason}`);
- shouldStop = true;
-
- if (!stream.destroyed) {
- stream.destroy();
- }
-
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
- };
-
- parseStream(stream, {headers: true})
- .on("headers", async (headers) => {
- if (!validateHeaders(headers)) {
- await cleanup("Header validation failed");
- return;
- }
- })
- .on("data", async (row) => {
- if (shouldStop) {
- return;
- }
-
- chunk.push(row);
-
- if (chunk.length >= chunkSize) {
- chunkNumber++;
-
- if (chunkNumber > maxChunks) {
- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);
- return;
- }
-
- try {
- await boss.send(processImportChunkQueue, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- });
- } catch (error) {
- console.error("Failed to send chunk to queue:", error);
- await cleanup("Error sending chunk to processing queue");
- }
-
- chunk = [];
- }
- })
- .on("end", async (rowCount: number) => {
- try {
- if (!shouldStop && chunk.length > 0) {
- chunkNumber++;
-
- await boss.send(processImportChunkQueue, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- });
- }
-
- if (!shouldStop) {
- console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
-
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
- }
- } catch (error) {
- console.error("Error during final CSV chunk processing:", error);
- await cleanup("Error during final chunk processing");
- }
- })
- .on("error", async (error) => {
- console.error("CSV parsing error:", error);
- await cleanup("CSV parsing error occurred");
- });
- } catch (error) {
- console.error("Error in importInitiationQueue worker:", error);
- // optional: notify, requeue, or cleanup
- }
-});
diff --git a/server/src/workers/processImportChunk.ts b/server/src/workers/processImportChunk.ts
deleted file mode 100644
index 5e4b0cfe7..000000000
--- a/server/src/workers/processImportChunk.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import boss from "../lib/boss.js";
-import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";
-import { ProcessImportChunkJob, processImportChunkQueue } from "../types/import.js";
-import { clickhouse } from "../db/clickhouse/clickhouse.js";
-import { Job } from "pg-boss";
-
-await boss.work(processImportChunkQueue, async (job: Job<ProcessImportChunkJob<UmamiEvent>>[]) => {
- try {
- const { site, importId, source, chunk, chunkNumber } = job[0].data;
-
- const dataMapper = getImportDataMapping(source);
- const transformedRecords = dataMapper.transform(chunk, site, importId);
-
- await clickhouse.insert({
- table: "events",
- values: transformedRecords,
- format: "JSONEachRow",
- });
-
- console.log(`Processed chunk ${chunkNumber} for import ${importId}`);
- } catch (error) {
- console.error("Error in processImportChunkQueue worker:", error);
- }
-});
-
-function getImportDataMapping(source: string) {
- switch (source) {
- case "umami":
- return new UmamiImportMapper();
- default:
- throw new Error(`Unsupported import source: ${source}`);
- }
-}