Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cui2fa902e5b5ab
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index e87282d4f..2f53c70a2 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -11,6 +11,20 @@ export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
const { site, importId, source, tempFilePath, organization } = job.data;
+ const validHeaders = (headers: string[]): boolean => {
+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {
+ return arr1.length === arr2.length &&
+ arr1.every((value, index) => value === arr2[index]);
+ };
+
+ switch (source) {
+ case "umami":
+ return arraysAreEqual(umamiHeaders, headers);
+ default:
+ return false;
+ }
+ };
+
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
@@ -25,75 +39,57 @@ export async function registerCsvParseWorker() {
const stream = fs.createReadStream(tempFilePath);
- const validHeaders = (headers: string[]): boolean => {
- const arraysAreEqual = (arr1: string[], arr2: string[]) => {
- return arr1.length === arr2.length &&
- arr1.every((value, index) => value === arr2[index]);
- };
-
- switch (source) {
- case "umami":
- return arraysAreEqual(umamiHeaders, headers);
- default:
- return false;
- }
- };
-
await ImportStatusManager.updateStatus(importId, "processing");
- return new Promise<void>((resolve, reject) => {
- parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true })
- .on("headers", (headers) => {
- if (!validHeaders(headers)) {
- reject(new Error(`Invalid ${source} headers`));
- return;
- }
- })
- .on("data", (row) => {
- if (rowsProcessed >= importableEvents) {
- stream.destroy();
- return;
- }
-
- chunk.push(row);
- rowsProcessed++;
+ const csvStream = parseStream(stream, {
+ headers: true,
+ maxRows: importableEvents,
+ discardUnmappedColumns: true,
+ ignoreEmpty: true
+ });
- if (chunk.length >= chunkSize) {
- boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk: [...chunk],
- });
+ await new Promise<void>((resolve, reject) => {
+ csvStream.once("headers", (headers) => {
+ if (!validHeaders(headers)) {
+ reject(new Error(`Invalid ${source} headers`));
+ return;
+ }
+ resolve();
+ });
+ csvStream.once("error", reject);
+ });
- chunk = [];
- }
- })
- .on("end", async () => {
- try {
- if (chunk.length > 0) {
- await boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk: [...chunk],
- });
- }
+ for await (const data of csvStream) {
+ if (rowsProcessed >= importableEvents) {
+ break;
+ }
- await boss.send(IMPORT_COMPLETION_QUEUE, {
- importId,
- totalEvents: rowsProcessed,
- });
+ chunk.push(data);
+ rowsProcessed++;
- await fs.promises.unlink(tempFilePath);
- resolve();
- } catch (error) {
- reject(error);
- }
- })
- .on("error", (error) => {
- reject(error);
+ if (chunk.length >= chunkSize) {
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk: [...chunk],
});
+ chunk = [];
+ }
+ }
+
+ if (chunk.length > 0) {
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk: [...chunk],
+ });
+ }
+
+ await boss.send(IMPORT_COMPLETION_QUEUE, {
+ importId,
+ totalEvents: rowsProcessed,
});
} catch (error) {
console.error("Error in CSV parse worker:", error);
@@ -102,8 +98,9 @@ export async function registerCsvParseWorker() {
"failed",
error instanceof Error ? error.message : "Unknown error occurred"
);
- await fs.promises.unlink(tempFilePath);
throw error;
+ } finally {
+ await fs.promises.unlink(tempFilePath);
}
});
}