Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cuicf381d67651c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 29767a1a7..56bd6fca8 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -11,19 +11,20 @@ export async function registerCsvParseWorker() {
await boss.work(CSV_PARSE_QUEUE, { batchSize: 1, pollingIntervalSeconds: 10 }, async ([ job ]: Job<CsvParseJob>[]) => {
const { site, importId, source, tempFilePath, organization } = job.data;
- const cleanup = async (reason: string, status: "failed" | "completed" = "failed") => {
- console.log(`Stopping CSV processing for ${importId}: ${reason}`);
-
- await fs.promises.unlink(tempFilePath);
- if (status === "failed") {
- await ImportStatusManager.updateStatus(importId, "failed", reason);
+ const cleanup = async () => {
+ try {
+ await fs.promises.unlink(tempFilePath);
+ console.log(`Deleted temporary file: ${tempFilePath}`);
+ } catch (error) {
+ console.error(`Failed to delete file ${tempFilePath}:`, error);
}
};
try {
const importableEvents = await ImportLimiter.countImportableEvents(organization);
if (importableEvents <= 0) {
- await cleanup("Event import limit reached");
+ await ImportStatusManager.updateStatus(importId, "failed", "Event import limit reached");
+ await cleanup();
return;
}
@@ -31,7 +32,6 @@ export async function registerCsvParseWorker() {
let chunkNumber = 0;
let chunk: UmamiEvent[] = [];
let rowsProcessed = 0;
- let pendingJobs: Promise<any>[] = [];
const stream = fs.createReadStream(tempFilePath);
@@ -49,31 +49,12 @@ export async function registerCsvParseWorker() {
}
};
- const queueChunk = (chunk: UmamiEvent[], finalChunk: boolean = false) => {
- chunkNumber++;
-
- const jobPromise = boss.send(DATA_INSERT_QUEUE, {
- site,
- importId,
- source,
- chunk,
- chunkNumber,
- finalChunk,
- });
-
- pendingJobs.push(jobPromise);
- console.log(`Queued chunk ${chunkNumber} with ${chunk.length} records (finalChunk: ${finalChunk})`);
-
- return jobPromise;
- };
-
await ImportStatusManager.updateStatus(importId, "processing");
return new Promise<void>((resolve, reject) => {
parseStream(stream, { headers: true, maxRows: importableEvents, discardUnmappedColumns: true, ignoreEmpty: true })
.on("headers", (headers) => {
if (!validHeaders(headers)) {
- cleanup(`Invalid ${source} headers`);
reject(new Error(`Invalid ${source} headers`));
return;
}
@@ -88,45 +69,58 @@ export async function registerCsvParseWorker() {
rowsProcessed++;
if (chunk.length >= chunkSize) {
- queueChunk([...chunk], false);
+ chunkNumber++;
+
+ boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk: [...chunk],
+ chunkNumber,
+ });
+
chunk = [];
}
})
.on("end", async () => {
try {
- // Queue any remaining chunk as the final chunk
if (chunk.length > 0) {
- queueChunk([...chunk], true);
- } else if (chunkNumber > 0) {
- // If no remaining chunk but we have processed chunks,
- // we need to mark the last one as final
- // This is a edge case - we'll let the data insert worker handle completion
- // based on the finalChunk flag of the last chunk sent
+ chunkNumber++;
+ await boss.send(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ source,
+ chunk: [...chunk],
+ chunkNumber,
+ });
}
- // Wait for all job queuing to complete (not the jobs themselves)
- await Promise.all(pendingJobs);
+ // Send completion job
+ // await boss.send(IMPORT_COMPLETION_QUEUE, {
+ // importId,
+ // totalChunks: chunkNumber,
+ // totalRecords: rowsProcessed,
+ // });
- console.log(`Successfully queued ${chunkNumber} chunks for processing`);
- await cleanup("All chunks queued successfully", "completed");
+ console.log(`Queued ${chunkNumber} chunks and completion job for import ${importId}`);
+ await cleanup();
resolve();
} catch (error) {
- console.error("Error queuing final chunks:", error);
- await cleanup(error instanceof Error ? error.message : "Unknown error");
reject(error);
}
})
- .on("error", async (error) => {
- console.error("CSV parsing error:", error);
- stream.destroy();
- await cleanup(error.message);
+ .on("error", (error) => {
reject(error);
});
});
} catch (error) {
console.error("Error in CSV parse worker:", error);
- const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
- await cleanup(errorMessage);
+ await ImportStatusManager.updateStatus(
+ importId,
+ "failed",
+ error instanceof Error ? error.message : "Unknown error occurred"
+ );
+ await cleanup();
throw error;
}
});