Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Edward Cui5d0cf04cbc98
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index accbee375..2e1dd4194 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -20,7 +20,6 @@ export async function registerCsvParseWorker() {
}
const chunkSize = 1000;
- let chunkNumber = 0;
let chunk: UmamiEvent[] = [];
let rowsProcessed = 0;
@@ -60,14 +59,11 @@ export async function registerCsvParseWorker() {
rowsProcessed++;
if (chunk.length >= chunkSize) {
- chunkNumber++;
-
boss.send(DATA_INSERT_QUEUE, {
site,
importId,
source,
chunk: [...chunk],
- chunkNumber,
});
chunk = [];
@@ -76,24 +72,20 @@ export async function registerCsvParseWorker() {
.on("end", async () => {
try {
if (chunk.length > 0) {
- chunkNumber++;
await boss.send(DATA_INSERT_QUEUE, {
site,
importId,
source,
chunk: [...chunk],
- chunkNumber,
});
}
// Send completion job
// await boss.send(IMPORT_COMPLETION_QUEUE, {
// importId,
- // totalChunks: chunkNumber,
// totalRecords: rowsProcessed,
// });
- console.log(`Queued ${chunkNumber} chunks and completion job for import ${importId}`);
await fs.promises.unlink(tempFilePath);
resolve();
} catch (error) {
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 92685fab1..253541fd7 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -7,7 +7,7 @@ import { ImportStatusManager } from "../importStatusManager.js";
export async function registerDataInsertWorker() {
await boss.work(DATA_INSERT_QUEUE, { batchSize: 1, pollingIntervalSeconds: 2 }, async ([ job ]: Job<DataInsertJob>[]) => {
- const { site, importId, source, chunk, chunkNumber } = job.data;
+ const { site, importId, source, chunk } = job.data;
try {
const getImportDataMapping = (source: string) => {
@@ -28,8 +28,6 @@ export async function registerDataInsertWorker() {
format: "JSONEachRow",
});
- console.log(`Successfully processed chunk ${chunkNumber} for import ${importId} (${transformedRecords.length} records)`);
-
await ImportStatusManager.updateProgress(
importId,
transformedRecords.length
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 32a5f7984..c19c82f11 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -17,5 +17,4 @@ export interface CsvParseJob extends ImportJob {
export interface DataInsertJob extends ImportJob {
chunk: UmamiEvent[];
- chunkNumber: number;
}