🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Better csvParseWorker.ts
Edward Cuic4d9998d87aa
diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.tsindex 9c5440342..fb7b26eaa 100644--- a/server/src/workers/csvParseWorker.ts+++ b/server/src/workers/csvParseWorker.ts@@ -3,54 +3,37 @@ import { parseStream } from "@fast-csv/parse";import boss from "../lib/boss.js";import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";import { Job } from "pg-boss";-import { umamiHeaders } from "./mappings/umami.js";-import { clickhouse } from "../db/clickhouse/clickhouse.js";-import { processResults } from "../api/analytics/utils.js";+import { UmamiEvent, umamiHeaders } from "./mappings/umami.js";+import { ImportStatusManager } from "../lib/importStatus.js";+import { ImportRateLimiter } from "../lib/rateLimiter.js";export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {try {const { site, importId, source, tempFilePath, organization } = job.data;- const query = await clickhouse.query({- query: `- SELECT count() AS count- FROM events- WHERE site_id = {siteId:UInt16}- AND import_id IS NOT NULL- `,- format: "JSONEachRow",- query_params: {- siteId: Number(site),- },- });- const result = await processResults<{ count: number }>(query);- const importedRowCount = result[0].count;-- const maxImportedRows = 1000000;- const remainingRows = maxImportedRows - importedRowCount;- if (remainingRows <= 0) {- console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);+ const importableEvents = await ImportRateLimiter.countImportableEvents(organization);+ if (importableEvents <= 0) {+ console.log(`No more events can be imported for importId: ${importId}.`);await fs.promises.unlink(tempFilePath);+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");return;}const chunkSize = 1000;- const maxChunks = Math.ceil(remainingRows / chunkSize);+ const maxChunks = Math.ceil(importableEvents / chunkSize);let chunkNumber = 0;- let chunk: any[] = [];+ let chunk: UmamiEvent[] = [];let shouldStop = false;const stream = fs.createReadStream(tempFilePath);- const arraysAreEqual = (arr1: string[], arr2: string[]) => {- if (arr1.length !== arr2.length) {- return false;- }- return arr1.every((value, index) => value === arr2[index]);- }+ const validHeaders = (headers: string[]): boolean => {+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {+ return arr1.length === arr2.length &&+ arr1.every((value, index) => value === arr2[index]);+ };- const validateHeaders = (headers: string[]): boolean => {switch (source) {case "umami":return arraysAreEqual(umamiHeaders, headers);@@ -59,8 +42,8 @@ export async function registerCsvParseWorker() {}};- const cleanup = async (reason: string) => {- console.log(`Stopping CSV processing: ${reason}`);+ const cleanup = async (reason: string, error?: unknown) => {+ console.log(`Stopping CSV processing for ${importId}: ${reason}`);shouldStop = true;if (!stream.destroyed) {@@ -75,10 +58,13 @@ export async function registerCsvParseWorker() {}};- parseStream(stream, {headers: true})+ await ImportStatusManager.updateStatus(importId, "processing");++ parseStream(stream, { headers: true, maxRows: importableEvents }).on("headers", async (headers) => {- if (!validateHeaders(headers)) {+ if (!validHeaders(headers)) {await cleanup("Header validation failed");+ await ImportStatusManager.updateStatus(importId, "failed", `Invalid ${source} headers`);return;}})@@ -87,13 +73,15 @@ export async function registerCsvParseWorker() {return;}+ // TODO: Validate data and skip invalid rowschunk.push(row);if (chunk.length >= chunkSize) {chunkNumber++;if (chunkNumber > maxChunks) {- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);+ await cleanup(`Maximum import limit reached`);+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");return;}@@ -104,10 +92,12 @@ export async function registerCsvParseWorker() {source,chunk,chunkNumber,+ finalChunk: chunkNumber === maxChunks,});} catch (error) {- console.error("Failed to send chunk to queue:", error);- await cleanup("Error sending chunk to processing queue");+ await cleanup("Error processing chunk");+ await ImportStatusManager.updateStatus(importId, "failed", "Error processing chunk");+ return;}chunk = [];@@ -124,31 +114,30 @@ export async function registerCsvParseWorker() {source,chunk,chunkNumber,+ finalChunk: true,});}- if (!shouldStop) {- console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);-- try {- await fs.promises.unlink(tempFilePath);- console.log(`Deleted temporary file: ${tempFilePath}`);- } catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);- }+ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);}} catch (error) {console.error("Error during final CSV chunk processing:", error);await cleanup("Error during final chunk processing");+ await ImportStatusManager.updateStatus(importId, "failed", "Error during final chunk");}}).on("error", async (error) => {console.error("CSV parsing error:", error);await cleanup("CSV parsing error occurred");+ await ImportStatusManager.updateStatus(importId, "failed", "CSV parsing error occurred");});} catch (error) {- console.error("Error in importInitiationQueue worker:", error);- // optional: notify, requeue, or cleanup+ console.error("Error in CSV parse worker:", error);+ // cleanup, throw, and mark as failed}});}