Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better csvParseWorker.ts

Edward Cuic4d9998d87aa
diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.ts
index 9c5440342..fb7b26eaa 100644
--- a/server/src/workers/csvParseWorker.ts
+++ b/server/src/workers/csvParseWorker.ts
@@ -3,54 +3,37 @@ import { parseStream } from "@fast-csv/parse";
import boss from "../lib/boss.js";
import { CSV_PARSE_QUEUE, CsvParseJob, DATA_INSERT_QUEUE } from "../types/import.js";
import { Job } from "pg-boss";
-import { umamiHeaders } from "./mappings/umami.js";
-import { clickhouse } from "../db/clickhouse/clickhouse.js";
-import { processResults } from "../api/analytics/utils.js";
+import { UmamiEvent, umamiHeaders } from "./mappings/umami.js";
+import { ImportStatusManager } from "../lib/importStatus.js";
+import { ImportRateLimiter } from "../lib/rateLimiter.js";
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
try {
const { site, importId, source, tempFilePath, organization } = job.data;
- const query = await clickhouse.query({
- query: `
- SELECT count() AS count
- FROM events
- WHERE site_id = {siteId:UInt16}
- AND import_id IS NOT NULL
- `,
- format: "JSONEachRow",
- query_params: {
- siteId: Number(site),
- },
- });
- const result = await processResults<{ count: number }>(query);
- const importedRowCount = result[0].count;
-
- const maxImportedRows = 1000000;
- const remainingRows = maxImportedRows - importedRowCount;
- if (remainingRows <= 0) {
- console.log(`No remaining rows can be imported (already imported: ${importedRowCount}).`);
+ const importableEvents = await ImportRateLimiter.countImportableEvents(organization);
+ if (importableEvents <= 0) {
+ console.log(`No more events can be imported for importId: ${importId}.`);
await fs.promises.unlink(tempFilePath);
+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
return;
}
const chunkSize = 1000;
- const maxChunks = Math.ceil(remainingRows / chunkSize);
+ const maxChunks = Math.ceil(importableEvents / chunkSize);
let chunkNumber = 0;
- let chunk: any[] = [];
+ let chunk: UmamiEvent[] = [];
let shouldStop = false;
const stream = fs.createReadStream(tempFilePath);
- const arraysAreEqual = (arr1: string[], arr2: string[]) => {
- if (arr1.length !== arr2.length) {
- return false;
- }
- return arr1.every((value, index) => value === arr2[index]);
- }
+ const validHeaders = (headers: string[]): boolean => {
+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {
+ return arr1.length === arr2.length &&
+ arr1.every((value, index) => value === arr2[index]);
+ };
- const validateHeaders = (headers: string[]): boolean => {
switch (source) {
case "umami":
return arraysAreEqual(umamiHeaders, headers);
@@ -59,8 +42,8 @@ export async function registerCsvParseWorker() {
}
};
- const cleanup = async (reason: string) => {
- console.log(`Stopping CSV processing: ${reason}`);
+ const cleanup = async (reason: string, error?: unknown) => {
+ console.log(`Stopping CSV processing for ${importId}: ${reason}`);
shouldStop = true;
if (!stream.destroyed) {
@@ -75,10 +58,13 @@ export async function registerCsvParseWorker() {
}
};
- parseStream(stream, {headers: true})
+ await ImportStatusManager.updateStatus(importId, "processing");
+
+ parseStream(stream, { headers: true, maxRows: importableEvents })
.on("headers", async (headers) => {
- if (!validateHeaders(headers)) {
+ if (!validHeaders(headers)) {
await cleanup("Header validation failed");
+ await ImportStatusManager.updateStatus(importId, "failed", `Invalid ${source} headers`);
return;
}
})
@@ -87,13 +73,15 @@ export async function registerCsvParseWorker() {
return;
}
+ // TODO: Validate data and skip invalid rows
chunk.push(row);
if (chunk.length >= chunkSize) {
chunkNumber++;
if (chunkNumber > maxChunks) {
- await cleanup(`Maximum import limit reached: ${maxImportedRows} rows total (already imported: ${importedRowCount})`);
+ await cleanup(`Maximum import limit reached`);
+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
return;
}
@@ -104,10 +92,12 @@ export async function registerCsvParseWorker() {
source,
chunk,
chunkNumber,
+ finalChunk: chunkNumber === maxChunks,
});
} catch (error) {
- console.error("Failed to send chunk to queue:", error);
- await cleanup("Error sending chunk to processing queue");
+ await cleanup("Error processing chunk");
+ await ImportStatusManager.updateStatus(importId, "failed", "Error processing chunk");
+ return;
}
chunk = [];
@@ -124,31 +114,30 @@ export async function registerCsvParseWorker() {
source,
chunk,
chunkNumber,
+ finalChunk: true,
});
}
- if (!shouldStop) {
- console.log(`Split file ${rowCount} rows into ${chunkNumber} chunks for ${site} in batch ${importId}.`);
-
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
}
} catch (error) {
console.error("Error during final CSV chunk processing:", error);
await cleanup("Error during final chunk processing");
+ await ImportStatusManager.updateStatus(importId, "failed", "Error during final chunk");
}
})
.on("error", async (error) => {
console.error("CSV parsing error:", error);
await cleanup("CSV parsing error occurred");
+ await ImportStatusManager.updateStatus(importId, "failed", "CSV parsing error occurred");
});
} catch (error) {
- console.error("Error in importInitiationQueue worker:", error);
- // optional: notify, requeue, or cleanup
+ console.error("Error in CSV parse worker:", error);
+ // cleanup, throw, and mark as failed
}
});
}