🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cuicf381d67651c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 29767a1a7..56bd6fca8 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -11,19 +11,20 @@ export async function registerCsvParseWorker() {await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {const { site, importId, source, tempFilePath, organization } = job.data;- const cleanup = async (reason: string, status: "failed" | "completed" = "failed") => {- console.log(`Stopping CSV processing for ${importId}: ${reason}`);-- await fs.promises.unlink(tempFilePath);- if (status === "failed") {- await ImportStatusManager.updateStatus(importId, "failed", reason);+ const cleanup = async () => {+ try {+ await fs.promises.unlink(tempFilePath);+ console.log(`Deleted temporary file: ${tempFilePath}`);+ } catch (error) {+ console.error(`Failed to delete file ${tempFilePath}:`, error);}};try {const importableEvents = await ImportLimiter.countImportableEvents(organization);if (importableEvents <= 0) {- await cleanup("Event import limit reached");+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");+ await cleanup();return;}@@ -31,7 +32,6 @@ export async function registerCsvParseWorker() {let chunkNumber = 0;let chunk: UmamiEvent[] = [];let rowsProcessed = 0;- let pendingJobs: Promise<any>[] = [];const stream = fs.createReadStream(tempFilePath);@@ -49,31 +49,12 @@ export async function registerCsvParseWorker() {}};- const queueChunk = (chunk: UmamiEvent[], finalChunk: boolean = false) => {- chunkNumber++;-- const jobPromise = boss.send(DATA_INSERT_QUEUE, {- site,- importId,- source,- chunk,- chunkNumber,- finalChunk,- });-- pendingJobs.push(jobPromise);- console.log(`Queued chunk ${chunkNumber} with ${chunk.length} records (finalChunk: ${finalChunk})`);-- return jobPromise;- };-await ImportStatusManager.updateStatus(importId, "processing");return new Promise<void>((resolve, reject) => {parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true }).on("headers", (headers) => {if (!validHeaders(headers)) {- cleanup(`Invalid ${source} headers`);reject(new Error(`Invalid ${source} headers`));return;}@@ -88,45 +69,58 @@ export async function registerCsvParseWorker() {rowsProcessed++;if (chunk.length >= chunkSize) {- queueChunk([...chunk], false);+ chunkNumber++;++ boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk: [...chunk],+ chunkNumber,+ });+chunk = [];}}).on("end", async () => {try {- // Queue any remaining chunk as the final chunkif (chunk.length > 0) {- queueChunk([...chunk], true);- } else if (chunkNumber > 0) {- // If no remaining chunk but we have processed chunks,- // we need to mark the last one as final- // This is a edge case - we'll let the data insert worker handle completion- // based on the finalChunk flag of the last chunk sent+ chunkNumber++;+ await boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk: [...chunk],+ chunkNumber,+ });}- // Wait for all job queuing to complete (not the jobs themselves)- await Promise.all(pendingJobs);+ // Send completion job+ // await boss.send(IMPORT_COMPLETION_QUEUE, {+ // importId,+ // totalChunks: chunkNumber,+ // totalRecords: rowsProcessed,+ // });- console.log(`Successfully queued ${chunkNumber} chunks for processing`);- await cleanup("All chunks queued successfully", "completed");+ console.log(`Queued ${chunkNumber} chunks and completion job for import ${importId}`);+ await cleanup();resolve();} catch (error) {- console.error("Error queuing final chunks:", error);- await cleanup(error instanceof Error ? error.message : "Unknown error");reject(error);}})- .on("error", async (error) => {- console.error("CSV parsing error:", error);- stream.destroy();- await cleanup(error.message);+ .on("error", (error) => {reject(error);});});} catch (error) {console.error("Error in CSV parse worker:", error);- const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";- await cleanup(errorMessage);+ await ImportStatusManager.updateStatus(+ importId,+ "failed",+ error instanceof Error ? error.message : "Unknown error occurred"+ );+ await cleanup();throw error;}});