Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Better error handling

Edward Cui62c4e9f6d5b7
diff --git a/server/src/workers/csvParseWorker.ts b/server/src/workers/csvParseWorker.ts
index fb7b26eaa..67c8e6326 100644
--- a/server/src/workers/csvParseWorker.ts
+++ b/server/src/workers/csvParseWorker.ts
@@ -9,14 +9,27 @@ import { ImportRateLimiter } from "../lib/rateLimiter.js";
export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
- try {
- const { site, importId, source, tempFilePath, organization } = job.data;
+ const { site, importId, source, tempFilePath, organization } = job.data;
+
+ const cleanup = async (reason: string, status: "failed" | "completed" = "failed") => {
+ console.log(`Stopping CSV processing for ${importId}: ${reason}`);
+
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
+ }
+ if (status === "failed") {
+ await ImportStatusManager.updateStatus(importId, "failed", reason);
+ }
+ };
+
+ try {
const importableEvents = await ImportRateLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
- console.log(`No more events can be imported for importId: ${importId}.`);
- await fs.promises.unlink(tempFilePath);
- await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
+ await cleanup("Event import limit reached");
return;
}
@@ -24,7 +37,6 @@ export async function registerCsvParseWorker() {
const maxChunks = Math.ceil(importableEvents / chunkSize);
let chunkNumber = 0;
let chunk: UmamiEvent[] = [];
- let shouldStop = false;
const stream = fs.createReadStream(tempFilePath);
@@ -42,102 +54,86 @@ export async function registerCsvParseWorker() {
}
};
- const cleanup = async (reason: string, error?: unknown) => {
- console.log(`Stopping CSV processing for ${importId}: ${reason}`);
- shouldStop = true;
-
- if (!stream.destroyed) {
- stream.destroy();
- }
-
- try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
- } catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
- }
- };
-
await ImportStatusManager.updateStatus(importId, "processing");
- parseStream(stream, { headers: true, maxRows: importableEvents })
- .on("headers", async (headers) => {
- if (!validHeaders(headers)) {
- await cleanup("Header validation failed");
- await ImportStatusManager.updateStatus(importId, "failed", `Invalid ${source} headers`);
- return;
- }
- })
- .on("data", async (row) => {
- if (shouldStop) {
- return;
- }
-
- // TODO: Validate data and skip invalid rows
- chunk.push(row);
-
- if (chunk.length >= chunkSize) {
- chunkNumber++;
-
- if (chunkNumber > maxChunks) {
- await cleanup(`Maximum import limit reached`);
- await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
- return;
- }
-
+ return new Promise<void>((resolve, reject) => {
+ parseStream(stream, { headers: true, maxRows: importableEvents })
+ .on("headers", async (headers) => {
try {
- await boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- finalChunk: chunkNumber === maxChunks,
- });
+ if (!validHeaders(headers)) {
+ stream.destroy();
+ await cleanup(`Invalid ${source} headers`);
+ reject(new Error(`Invalid ${source} headers`));
+ return;
+ }
} catch (error) {
- await cleanup("Error processing chunk");
- await ImportStatusManager.updateStatus(importId, "failed", "Error processing chunk");
- return;
+ stream.destroy();
+ reject(error);
}
-
- chunk = [];
- }
- })
- .on("end", async (rowCount: number) => {
- try {
- if (!shouldStop && chunk.length > 0) {
- chunkNumber++;
-
- await boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- finalChunk: true,
- });
+ })
+ .on("data", async (row) => {
+ try {
+ // TODO: Validate data and skip invalid rows
+ chunk.push(row);
+
+ if (chunk.length >= chunkSize) {
+ chunkNumber++;
+
+ if (chunkNumber > maxChunks) {
+ stream.destroy();
+ await cleanup("Event import limit reached");
+ reject(new Error("Event import limit reached"));
+ return;
+ }
+
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ finalChunk: chunkNumber === maxChunks,
+ });
+
+ chunk = [];
+ }
+ } catch (error) {
+ stream.destroy();
+ reject(error);
}
-
+ })
+ .on("end", async () => {
try {
- await fs.promises.unlink(tempFilePath);
- console.log(`Deleted temporary file: ${tempFilePath}`);
+ // Process any remaining chunk
+ if (chunk.length > 0) {
+ chunkNumber++;
+
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk,
+ chunkNumber,
+ finalChunk: true,
+ });
+ }
+
+ await cleanup("Processing completed", "completed");
+ resolve();
} catch (error) {
- console.error(`Failed to delete file ${tempFilePath}:`, error);
+ reject(error);
}
- } catch (error) {
- console.error("Error during final CSV chunk processing:", error);
- await cleanup("Error during final chunk processing");
- await ImportStatusManager.updateStatus(importId, "failed", "Error during final chunk");
- }
- })
- .on("error", async (error) => {
- console.error("CSV parsing error:", error);
- await cleanup("CSV parsing error occurred");
- await ImportStatusManager.updateStatus(importId, "failed", "CSV parsing error occurred");
- });
+ })
+ .on("error", (error) => {
+ stream.destroy();
+ reject(error);
+ });
+ });
+
} catch (error) {
console.error("Error in CSV parse worker:", error);
- // cleanup, throw, and mark as failed
+ await cleanup(error instanceof Error ? error.message : "Unknown error occurred");
+ throw error;
}
});
}