🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cuiadafe495ab5c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex c8fd919fd..29767a1a7 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -14,13 +14,7 @@ export async function registerCsvParseWorker() {const cleanup = async (reason: string, status: "failed" | "completed" = "failed") => {console.log(`Stopping CSV processing for ${importId}: ${reason}`);- try {- await fs.promises.unlink(tempFilePath);- console.log(`Deleted temporary file: ${tempFilePath}`);- } catch (error) {- console.error(`Failed to delete file ${tempFilePath}:`, error);- }-+ await fs.promises.unlink(tempFilePath);if (status === "failed") {await ImportStatusManager.updateStatus(importId, "failed", reason);}@@ -34,9 +28,10 @@ export async function registerCsvParseWorker() {}const chunkSize = 1000;- const maxChunks = Math.ceil(importableEvents / chunkSize);let chunkNumber = 0;let chunk: UmamiEvent[] = [];+ let rowsProcessed = 0;+ let pendingJobs: Promise<any>[] = [];const stream = fs.createReadStream(tempFilePath);@@ -54,84 +49,84 @@ export async function registerCsvParseWorker() {}};+ const queueChunk = (chunk: UmamiEvent[], finalChunk: boolean = false) => {+ chunkNumber++;++ const jobPromise = boss.send(DATA_INSERT_QUEUE, {+ site,+ importId,+ source,+ chunk,+ chunkNumber,+ finalChunk,+ });++ pendingJobs.push(jobPromise);+ console.log(`Queued chunk ${chunkNumber} with ${chunk.length} records (finalChunk: ${finalChunk})`);++ return jobPromise;+ };+await ImportStatusManager.updateStatus(importId, "processing");return new Promise<void>((resolve, reject) => {- parseStream(stream, { headers: true, maxRows: importableEvents })- .on("headers", async (headers) => {- try {- if (!validHeaders(headers)) {- stream.destroy();- await cleanup(`Invalid ${source} headers`);- reject(new Error(`Invalid ${source} headers`));- return;- }- } catch (error) {- stream.destroy();- reject(error);+ parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true })+ .on("headers", (headers) => {+ if (!validHeaders(headers)) {+ cleanup(`Invalid ${source} headers`);+ reject(new Error(`Invalid ${source} headers`));+ return;}})- .on("data", async (row) => {- try {- // TODO: Validate data and skip invalid rows- chunk.push(row);-- if (chunk.length >= chunkSize) {- chunkNumber++;-- if (chunkNumber > maxChunks) {- stream.destroy();- await cleanup("Event import limit reached");- reject(new Error("Event import limit reached"));- return;- }-- await boss.send(DATA_INSERT_QUEUE, {- site,- importId,- source,- chunk,- chunkNumber,- finalChunk: chunkNumber === maxChunks,- });-- chunk = [];- }- } catch (error) {+ .on("data", (row) => {+ if (rowsProcessed >= importableEvents) {stream.destroy();- reject(error);+ return;+ }++ chunk.push(row);+ rowsProcessed++;++ if (chunk.length >= chunkSize) {+ queueChunk([...chunk], false);+ chunk = [];}}).on("end", async () => {try {- // Process any remaining chunk+ // Queue any remaining chunk as the final chunkif (chunk.length > 0) {- chunkNumber++;-- await boss.send(DATA_INSERT_QUEUE, {- site,- importId,- source,- chunk,- chunkNumber,- finalChunk: true,- });+ queueChunk([...chunk], true);+ } else if (chunkNumber > 0) {+ // If no remaining chunk but we have processed chunks,+ // we need to mark the last one as final+ // This is a edge case - we'll let the data insert worker handle completion+ // based on the finalChunk flag of the last chunk sent}- await cleanup("Processing completed", "completed");+ // Wait for all job queuing to complete (not the jobs themselves)+ await Promise.all(pendingJobs);++ console.log(`Successfully queued ${chunkNumber} chunks for processing`);+ await cleanup("All chunks queued successfully", "completed");resolve();} catch (error) {+ console.error("Error queuing final chunks:", error);+ await cleanup(error instanceof Error ? error.message : "Unknown error");reject(error);}})- .on("error", (error) => {+ .on("error", async (error) => {+ console.error("CSV parsing error:", error);stream.destroy();+ await cleanup(error.message);reject(error);});});} catch (error) {console.error("Error in CSV parse worker:", error);- await cleanup(error instanceof Error ? error.message : "Unknown error occurred");+ const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";+ await cleanup(errorMessage);throw error;}});