🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Edward Cui5d0cf04cbc98
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex accbee375..2e1dd4194 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -20,7 +20,6 @@ export async function registerCsvParseWorker() {}const chunkSize = 1000;- let chunkNumber = 0;let chunk: UmamiEvent[] = [];let rowsProcessed = 0;@@ -60,14 +59,11 @@ export async function registerCsvParseWorker() {rowsProcessed++;if (chunk.length >= chunkSize) {- chunkNumber++;-boss.send(DATA_INSERT_QUEUE, {site,importId,source,chunk: [...chunk],- chunkNumber,});chunk = [];@@ -76,24 +72,20 @@ export async function registerCsvParseWorker() {.on("end", async () => {try {if (chunk.length > 0) {- chunkNumber++;await boss.send(DATA_INSERT_QUEUE, {site,importId,source,chunk: [...chunk],- chunkNumber,});}// Send completion job// await boss.send(IMPORT_COMPLETION_QUEUE, {// importId,- // totalChunks: chunkNumber,// totalRecords: rowsProcessed,// });- console.log(`Queued ${chunkNumber} chunks and completion job for import ${importId}`);await fs.promises.unlink(tempFilePath);resolve();} catch (error) {diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 92685fab1..253541fd7 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -7,7 +7,7 @@ import { ImportStatusManager } from "../importStatusManager.js";export async function registerDataInsertWorker() {await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {- const { site, importId, source, chunk, chunkNumber } = job.data;+ const { site, importId, source, chunk } = job.data;try {const getImportDataMapping = (source: string) => {@@ -28,8 +28,6 @@ export async function registerDataInsertWorker() {format: "JSONEachRow",});- console.log(`Successfully processed chunk ${chunkNumber} for import ${importId} (${transformedRecords.length} records)`);-await ImportStatusManager.updateProgress(importId,transformedRecords.lengthdiff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex 32a5f7984..c19c82f11 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -17,5 +17,4 @@ export interface CsvParseJob extends ImportJob {export interface DataInsertJob extends ImportJob {chunk: UmamiEvent[];- chunkNumber: number;}