🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cui2fa902e5b5ab
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex e87282d4f..2f53c70a2 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -11,6 +11,20 @@ export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {const { site, importId, source, tempFilePath, organization } = job.data;+ const validHeaders = (headers: string[]): boolean => {+ const arraysAreEqual = (arr1: string[], arr2: string[]) => {+ return arr1.length === arr2.length &&+ arr1.every((value, index) => value === arr2[index]);+ };++ switch (source) {+ case "umami":+ return arraysAreEqual(umamiHeaders, headers);+ default:+ return false;+ }+ };+try {const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {@@ -25,75 +39,57 @@ export async function registerCsvParseWorker() {const stream = fs.createReadStream(tempFilePath);- const validHeaders = (headers: string[]): boolean => {- const arraysAreEqual = (arr1: string[], arr2: string[]) => {- return arr1.length === arr2.length &&- arr1.every((value, index) => value === arr2[index]);- };-- switch (source) {- case "umami":- return arraysAreEqual(umamiHeaders, headers);- default:- return false;- }- };-await ImportStatusManager.updateStatus(importId, "processing");- return new Promise<void>((resolve, reject) => {- parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true })- .on("headers", (headers) => {- if (!validHeaders(headers)) {- reject(new Error(`Invalid ${source} headers`));- return;- }- })- .on("data", (row) => {- if (rowsProcessed >= importableEvents) {- stream.destroy();- return;- }-- chunk.push(row);- rowsProcessed++;+ const csvStream = parseStream(stream, {+ headers: true,+ maxRows: importableEvents,+ discardUnmappedColumns: true,+ ignoreEmpty: true+ });- if (chunk.length >= chunkSize) {- boss.send(DATA_INSERT_QUEUE, {- site,- importId,- source,- chunk: [...chunk],- });+ await new Promise<void>((resolve, reject) => {+ csvStream.once("headers", (headers) => {+ if (!validHeaders(headers)) {+ reject(new Error(`Invalid ${source} headers`));+ return;+ }+ resolve();+ });+ csvStream.once("error", reject);+ });- chunk = [];- }- })- .on("end", async () => {- try {- if (chunk.length > 0) {- await boss.send(DATA_INSERT_QUEUE, {- site,- importId,- source,- chunk: [...chunk],- });- }+ for await (const data of csvStream) {+ if (rowsProcessed >= importableEvents) {+ break;+ }- await boss.send(IMPORT_COMPLETION_QUEUE, {- importId,- totalEvents: rowsProcessed,- });+ chunk.push(data);+ rowsProcessed++;- await fs.promises.unlink(tempFilePath);- resolve();- } catch (error) {- reject(error);- }- })- .on("error", (error) => {- reject(error);+ if (chunk.length >= chunkSize) {+ await boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk: [...chunk],});+ chunk = [];+ }+ }++ if (chunk.length > 0) {+ await boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk: [...chunk],+ });+ }++ await boss.send(IMPORT_COMPLETION_QUEUE, {+ importId,+ totalEvents: rowsProcessed,});} catch (error) {console.error("Error in CSV parse worker:", error);@@ -102,8 +98,9 @@ export async function registerCsvParseWorker() {"failed",error instanceof Error ? error.message : "Unknown error occurred");- await fs.promises.unlink(tempFilePath);throw error;+ } finally {+ await fs.promises.unlink(tempFilePath);}});}