🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor and register workers
Edward Cui4a4a82a827cd
diff --git a/server/src/api/importData.ts b/server/src/api/importData.tsindex 95ecbef9c..203249969 100644--- a/server/src/api/importData.ts+++ b/server/src/api/importData.ts@@ -6,7 +6,7 @@ import { z } from "zod";import crypto from "crypto";import boss from "../lib/boss.js";import { getUserHasAccessToSite } from "../lib/auth-utils.js";-import { importInitiationQueue } from "../types/import.js";+import { CSV_PARSE_QUEUE } from "../types/import.js";const IMPORT_DIR = "/tmp/imports";@@ -73,7 +73,7 @@ export async function importData(}try {- await boss.send(importInitiationQueue, {+ await boss.send(CSV_PARSE_QUEUE, {tempFilePath,organization,site,diff --git a/server/src/index.ts b/server/src/index.tsindex 616f45610..0ddbbcadb 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -71,6 +71,8 @@ import { trackEvent } from "./tracker/trackEvent.js";import { extractSiteId, isSitePublic } from "./utils.js";import { importData } from "./api/importData.js";import boss from "./lib/boss.js";+import { registerCsvParseWorker } from "./workers/csvParseWorker.js";+import { registerDataInsertWorker } from "./workers/dataInsertWorker.js";const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);@@ -337,6 +339,8 @@ const start = async () => {await siteConfig.loadSiteConfigs();await boss.start();+ await registerCsvParseWorker();+ await registerDataInsertWorker();// Start the serverawait server.listen({ port: 3001, host: "0.0.0.0" });diff --git a/server/src/types/import.ts b/server/src/types/import.tsindex 1390185d7..cfc73a783 100644--- a/server/src/types/import.ts+++ b/server/src/types/import.ts@@ -1,6 +1,6 @@-export const importInitiationQueue = "import-initiation";+export const CSV_PARSE_QUEUE = "csv-parse";-export const processImportChunkQueue = "process-import-chunk";+export const DATA_INSERT_QUEUE = "data-insert";interface ImportJob {site: string;@@ -8,12 +8,12 @@ interface ImportJob {source: "umami";}-export interface ImportInitiationJob extends ImportJob {+export interface CsvParseJob extends ImportJob {tempFilePath: string;organization: string;}-export interface ProcessImportChunkJob<T> extends ImportJob {+export interface DataInsertJob<T> extends ImportJob {chunk: T[];chunkNumber: number;}diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.tsnew file mode 100644index 000000000..1b072468a--- /dev/null+++ b/server/src/workers/csvParseWorker.ts@@ -0,0 +1,154 @@+import fs from "fs";+import { parseStream } from "@fast-csv/parse";+import boss from "../lib/boss.js";+import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";+import { Job } from "pg-boss";+import { umamiHeaders } from "./mappings/umami.js";+import { clickhouse } from "../db/clickhouse/clickhouse.js";+import { processResults } from "../api/analytics/utils.js";++export async function registerCsvParseWorker() {+ await boss.work(CSV_PARSE_QUEUE, async (job: Job<CsvParseJob>[]) => {+ try {+ const {site, importId, source, tempFilePath, organization} = job[0].data;++ const query = await clickhouse.query({+ query: `+ SELECT count() AS count+ FROM events+ WHERE site_id = {siteId:UInt16}+ AND import_id IS NOT NULL+ `,+ format: "JSONEachRow",+ query_params: {+ siteId: Number(site),+ },+ });+ const result = await processResults<{ count: number }>(query);+ const importedRowCount = result[0].count;++ const maxImportedRows = 1000000;+ const remainingRows = maxImportedRows - importedRowCount;+ if (remainingRows <= 0) {+ console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);+ await fs.promises.unlink(tempFilePath);+ return;+ }++ const chunkSize = 1000;+ const maxChunks = Math.ceil(remainingRows / chunkSize);+ let chunkNumber = 0;+ let chunk: any[] = [];+ let shouldStop = false;++ const stream = fs.createReadStream(tempFilePath);++ const arraysAreEqual = (arr1: string[], arr2: string[]) => {+ if (arr1.length !== arr2.length) {+ return false;+ }+ return arr1.every((value, index) => value === arr2[index]);+ }++ const validateHeaders = (headers: string[]): boolean => {+ switch (source) {+ case "umami":+ return arraysAreEqual(umamiHeaders, headers);+ default:+ return false;+ }+ };++ const cleanup = async (reason: string) => {+ console.log(`Stopping CSV processing: ${reason}`);+ shouldStop = true;++ if (!stream.destroyed) {+ stream.destroy();+ }++ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ };++ parseStream(stream, {headers: true})+ .on("headers", async (headers) => {+ if (!validateHeaders(headers)) {+ await cleanup("Header validation failed");+ return;+ }+ })+ .on("data", async (row) => {+ if (shouldStop) {+ return;+ }++ chunk.push(row);++ if (chunk.length >= chunkSize) {+ chunkNumber++;++ if (chunkNumber > maxChunks) {+ await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);+ return;+ }++ try {+ await boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk,+ chunkNumber,+ });+ } catch (error) {+ console.error("Failed to send chunk to queue:", error);+ await cleanup("Error sending chunk to processing queue");+ }++ chunk = [];+ }+ })+ .on("end", async (rowCount: number) => {+ try {+ if (!shouldStop && chunk.length > 0) {+ chunkNumber++;++ await boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk,+ chunkNumber,+ });+ }++ if (!shouldStop) {+ console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);++ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);+ }+ }+ } catch (error) {+ console.error("Error during final CSV chunk processing:", error);+ await cleanup("Error during final chunk processing");+ }+ })+ .on("error", async (error) => {+ console.error("CSV parsing error:", error);+ await cleanup("CSV parsing error occurred");+ });+ } catch (error) {+ console.error("Error in importInitiationQueue worker:", error);+ // optional: notify, requeue, or cleanup+ }+ });+}diff --git a/server/src/workers/dataInsertWorker.ts b/server/src/workers/dataInsertWorker.tsnew file mode 100644index 000000000..f154e7ad4--- /dev/null+++ b/server/src/workers/dataInsertWorker.ts@@ -0,0 +1,35 @@+import boss from "../lib/boss.js";+import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";+import { DataInsertJob, DATA_INSERT_QUEUE } from "../types/import.js";+import { clickhouse } from "../db/clickhouse/clickhouse.js";+import { Job } from "pg-boss";++export async function registerDataInsertWorker() {+ await boss.work(DATA_INSERT_QUEUE, async (job: Job<DataInsertJob<UmamiEvent>>[]) => {+ try {+ const {site, importId, source, chunk, chunkNumber} = job[0].data;++ const dataMapper = getImportDataMapping(source);+ const transformedRecords = dataMapper.transform(chunk, site, importId);++ await clickhouse.insert({+ table: "events",+ values: transformedRecords,+ format: "JSONEachRow",+ });++ console.log(`Processed chunk ${chunkNumber} for import ${importId}`);+ } catch (error) {+ console.error("Error in processImportChunkQueue worker:", error);+ }+ });++ function getImportDataMapping(source: string) {+ switch (source) {+ case "umami":+ return new UmamiImportMapper();+ default:+ throw new Error(`Unsupported import source: ${source}`);+ }+ }+}diff --git a/server/src/workers/importInitiation.ts b/server/src/workers/importInitiation.tsdeleted file mode 100644index c84a9c0e7..000000000--- a/server/src/workers/importInitiation.ts+++ /dev/null@@ -1,152 +0,0 @@-import fs from "fs";-import { parseStream } from "@fast-csv/parse";-import boss from "../lib/boss.js";-import { importInitiationQueue, ImportInitiationJob, processImportChunkQueue } from "../types/import.js";-import { Job } from "pg-boss";-import { umamiHeaders } from "./mappings/umami.js";-import { clickhouse } from "../db/clickhouse/clickhouse.js";-import { processResults } from "../api/analytics/utils.js";--await boss.work(importInitiationQueue, async (job: Job<ImportInitiationJob>[]) => {- try {- const { site, importId, source, tempFilePath, organization } = job[0].data;-- const query = await clickhouse.query({- query: `- SELECT count() AS count- FROM events- WHERE site_id = {siteId:UInt16}- AND import_id IS NOT NULL- `,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- });- const result = await processResults<{ count: number }>(query);- const importedRowCount = result[0].count;-- const maxImportedRows = 1000000;- const remainingRows = maxImportedRows - importedRowCount;- if (remainingRows <= 0) {- console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);- await fs.promises.unlink(tempFilePath);- return;- }-- const chunkSize = 1000;- const maxChunks = Math.ceil(remainingRows / chunkSize);- let chunkNumber = 0;- let chunk: any[] = [];- let shouldStop = false;-- const stream = fs.createReadStream(tempFilePath);-- const arraysAreEqual = (arr1: string[], arr2: string[]) => {- if (arr1.length !== arr2.length) {- return false;- }- return arr1.every((value, index) => value === arr2[index]);- }-- const validateHeaders = (headers: string[]): boolean => {- switch (source) {- case "umami":- return arraysAreEqual(umamiHeaders, headers);- default:- return false;- }- };-- const cleanup = async (reason: string) => {- console.log(`Stopping CSV processing: ${reason}`);- shouldStop = true;-- if (!stream.destroyed) {- stream.destroy();- }-- try {- await fs.promises.unlink(tempFilePath);- console.log(`Deleted temporary file: ${tempFilePath}`);- } catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);- }- };-- parseStream(stream, {headers: true})- .on("headers", async (headers) => {- if (!validateHeaders(headers)) {- await cleanup("Header validation failed");- return;- }- })- .on("data", async (row) => {- if (shouldStop) {- return;- }-- chunk.push(row);-- if (chunk.length >= chunkSize) {- chunkNumber++;-- if (chunkNumber > maxChunks) {- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);- return;- }-- try {- await boss.send(processImportChunkQueue, {- site,- importId,- source,- chunk,- chunkNumber,- });- } catch (error) {- console.error("Failed to send chunk to queue:", error);- await cleanup("Error sending chunk to processing queue");- }-- chunk = [];- }- })- .on("end", async (rowCount: number) => {- try {- if (!shouldStop && chunk.length > 0) {- chunkNumber++;-- await boss.send(processImportChunkQueue, {- site,- importId,- source,- chunk,- chunkNumber,- });- }-- if (!shouldStop) {- console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);-- try {- await fs.promises.unlink(tempFilePath);- console.log(`Deleted temporary file: ${tempFilePath}`);- } catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);- }- }- } catch (error) {- console.error("Error during final CSV chunk processing:", error);- await cleanup("Error during final chunk processing");- }- })- .on("error", async (error) => {- console.error("CSV parsing error:", error);- await cleanup("CSV parsing error occurred");- });- } catch (error) {- console.error("Error in importInitiationQueue worker:", error);- // optional: notify, requeue, or cleanup- }-});diff --git a/server/src/workers/processImportChunk.ts b/server/src/workers/processImportChunk.tsdeleted file mode 100644index 5e4b0cfe7..000000000--- a/server/src/workers/processImportChunk.ts+++ /dev/null@@ -1,33 +0,0 @@-import boss from "../lib/boss.js";-import { UmamiEvent, UmamiImportMapper } from "./mappings/umami.js";-import { ProcessImportChunkJob, processImportChunkQueue } from "../types/import.js";-import { clickhouse } from "../db/clickhouse/clickhouse.js";-import { Job } from "pg-boss";--await boss.work(processImportChunkQueue, async (job: Job<ProcessImportChunkJob<UmamiEvent>>[]) => {- try {- const { site, importId, source, chunk, chunkNumber } = job[0].data;-- const dataMapper = getImportDataMapping(source);- const transformedRecords = dataMapper.transform(chunk, site, importId);-- await clickhouse.insert({- table: "events",- values: transformedRecords,- format: "JSONEachRow",- });-- console.log(`Processed chunk ${chunkNumber} for import ${importId}`);- } catch (error) {- console.error("Error in processImportChunkQueue worker:", error);- }-});--function getImportDataMapping(source: string) {- switch (source) {- case "umami":- return new UmamiImportMapper();- default:- throw new Error(`Unsupported import source: ${source}`);- }-}