Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cuiadafe495ab5c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index c8fd919fd..29767a1a7 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -14,13 +14,7 @@ export async function registerCsvParseWorker() {
const cleanup = async (reason: string, status: "failed" | "completed" = "failed") => {
console.log(`Stopping CSV processing for ${importId}: ${reason}`);
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
-
+ await fs.promises.unlink(tempFilePath);
if (status === "failed") {
await ImportStatusManager.updateStatus(importId, "failed", reason);
}
@@ -34,9 +28,10 @@ export async function registerCsvParseWorker() {
}
const chunkSize = 1000;
- const maxChunks = Math.ceil(importableEvents / chunkSize);
let chunkNumber = 0;
let chunk: UmamiEvent[] = [];
+ let rowsProcessed = 0;
+ let pendingJobs: Promise<any>[] = [];
const stream = fs.createReadStream(tempFilePath);
@@ -54,84 +49,84 @@ export async function registerCsvParseWorker() {
}
};
+ const queueChunk = (chunk: UmamiEvent[], finalChunk: boolean = false) => {
+ chunkNumber++;
+
+ const jobPromise = boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ finalChunk,
+ });
+
+ pendingJobs.push(jobPromise);
+ console.log(`Queued chunk ${chunkNumber} with ${chunk.length} records (finalChunk: ${finalChunk})`);
+
+ return jobPromise;
+ };
+
await ImportStatusManager.updateStatus(importId, "processing");
return new Promise<void>((resolve, reject) => {
- parseStream(stream, { headers: true, maxRows: importableEvents })
- .on("headers", async (headers) => {
- try {
- if (!validHeaders(headers)) {
- stream.destroy();
- await cleanup(`Invalid ${source} headers`);
- reject(new Error(`Invalid ${source} headers`));
- return;
- }
- } catch (error) {
- stream.destroy();
- reject(error);
+ parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true })
+ .on("headers", (headers) => {
+ if (!validHeaders(headers)) {
+ cleanup(`Invalid ${source} headers`);
+ reject(new Error(`Invalid ${source} headers`));
+ return;
}
})
- .on("data", async (row) => {
- try {
- // TODO: Validate data and skip invalid rows
- chunk.push(row);
-
- if (chunk.length >= chunkSize) {
- chunkNumber++;
-
- if (chunkNumber > maxChunks) {
- stream.destroy();
- await cleanup("Event import limit reached");
- reject(new Error("Event import limit reached"));
- return;
- }
-
- await boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- finalChunk: chunkNumber === maxChunks,
- });
-
- chunk = [];
- }
- } catch (error) {
+ .on("data", (row) => {
+ if (rowsProcessed >= importableEvents) {
stream.destroy();
- reject(error);
+ return;
+ }
+
+ chunk.push(row);
+ rowsProcessed++;
+
+ if (chunk.length >= chunkSize) {
+ queueChunk([...chunk], false);
+ chunk = [];
}
})
.on("end", async () => {
try {
- // Process any remaining chunk
+ // Queue any remaining chunk as the final chunk
if (chunk.length > 0) {
- chunkNumber++;
-
- await boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- finalChunk: true,
- });
+ queueChunk([...chunk], true);
+ } else if (chunkNumber > 0) {
+ // If no remaining chunk but we have processed chunks,
+ // we need to mark the last one as final
+ // This is a edge case - we'll let the data insert worker handle completion
+ // based on the finalChunk flag of the last chunk sent
}
- await cleanup("Processing completed", "completed");
+ // Wait for all job queuing to complete (not the jobs themselves)
+ await Promise.all(pendingJobs);
+
+ console.log(`Successfully queued ${chunkNumber} chunks for processing`);
+ await cleanup("All chunks queued successfully", "completed");
resolve();
} catch (error) {
+ console.error("Error queuing final chunks:", error);
+ await cleanup(error instanceof Error ? error.message : "Unknown error");
reject(error);
}
})
- .on("error", (error) => {
+ .on("error", async (error) => {
+ console.error("CSV parsing error:", error);
stream.destroy();
+ await cleanup(error.message);
reject(error);
});
});
} catch (error) {
console.error("Error in CSV parse worker:", error);
- await cleanup(error instanceof Error ? error.message : "Unknown error occurred");
+ const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
+ await cleanup(errorMessage);
throw error;
}
});