Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cuib90ba9f8f145
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index effae9fd2..388dd8235 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -21,24 +21,18 @@ export async function registerCsvParseWorker() {
const chunkSize = 1000;
let chunk: UmamiEvent[] = [];
- let rowsProcessed = 0;
const stream = fs.createReadStream(tempFilePath).pipe(parse({
headers: umamiHeaders,
renameHeaders: true,
ignoreEmpty: true,
- // maxRows: importableEvents,
+ maxRows: importableEvents,
}));
await ImportStatusManager.updateStatus(importId, "processing");
for await (const data of stream) {
- if (rowsProcessed >= importableEvents) {
- break;
- }
-
chunk.push(data);
- rowsProcessed++;
if (chunk.length >= chunkSize) {
await boss.send(DATA_INSERT_QUEUE, {
@@ -62,7 +56,6 @@ export async function registerCsvParseWorker() {
await boss.send(IMPORT_COMPLETION_QUEUE, {
importId,
- totalEvents: rowsProcessed,
});
} catch (error) {
console.error("Error in CSV parse worker:", error);
diff --git a/server/src/services/import/workers/importCompletionWorker.ts b/server/src/services/import/workers/importCompletionWorker.ts
index 62a4a4a8e..bd928ef13 100644
--- a/server/src/services/import/workers/importCompletionWorker.ts
+++ b/server/src/services/import/workers/importCompletionWorker.ts
@@ -5,11 +5,11 @@ import { ImportStatusManager } from "../importStatusManager.js";
export async function registerImportCompletionWorker() {
await boss.work(IMPORT_COMPLETION_QUEUE, { batchSize: 1, pollingIntervalSeconds: 30 }, async ([ job ]: Job<ImportCompletionJob>[]) => {
- const { importId, totalEvents } = job.data;
+ const { importId } = job.data;
try {
await ImportStatusManager.updateStatus(importId, "completed");
- console.log(`Import ${importId} completed: ${totalEvents} events`);
+ console.log(`Import ${importId} completed`);
} catch (error) {
console.error(`Failed to mark import ${importId} as completed:`, error);
await ImportStatusManager.updateStatus(importId, "failed", "Failed to complete import");
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 0279a3438..c0c25847b 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -23,5 +23,4 @@ export interface DataInsertJob extends ImportJob {
export interface ImportCompletionJob {
importId: string;
- totalEvents: number;
}