Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify data insert job

Edward Cui54c1d9fefd6c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index fe1038d34..0736cd9d9 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -94,7 +94,6 @@ export async function registerCsvParseWorker() {
let totalAccepted = 0;
let totalSkippedQuota = 0;
let totalSkippedDate = 0;
- let chunksSent = 0;
stream = isR2Storage
? await createR2FileStream(storageLocation, platform)
@@ -145,10 +144,8 @@ export async function registerCsvParseWorker() {
importId,
platform,
chunk,
- chunkNumber: chunksSent,
allChunksSent: false,
});
- chunksSent++;
chunk = [];
}
}
@@ -180,10 +177,8 @@ export async function registerCsvParseWorker() {
importId,
platform,
chunk,
- chunkNumber: chunksSent,
allChunksSent: false,
});
- chunksSent++;
}
// Send finalization signal with total chunk count
@@ -192,7 +187,6 @@ export async function registerCsvParseWorker() {
importId,
platform,
chunk: [],
- totalChunks: chunksSent,
allChunksSent: true,
});
} catch (error) {
diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.ts
index 407242af4..0ee20dbe7 100644
--- a/server/src/services/import/workers/dataInsertWorker.ts
+++ b/server/src/services/import/workers/dataInsertWorker.ts
@@ -17,13 +17,11 @@ export async function registerDataInsertWorker() {
const jobQueue = getJobQueue();
await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {
- const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job;
+ const { site, importId, platform, chunk, allChunksSent } = job;
- // Handle finalization signal
if (allChunksSent) {
try {
await updateImportStatus(importId, "completed");
- console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);
return;
} catch (error) {
console.error(`[Import ${importId}] Failed to mark as completed:`, error);
@@ -38,7 +36,6 @@ export async function registerDataInsertWorker() {
}
}
- // Process data chunk
try {
const dataMapper = getImportDataMapping(platform);
const transformedRecords = dataMapper.transform(chunk, site, importId);
@@ -60,8 +57,6 @@ export async function registerDataInsertWorker() {
);
// Don't throw - data is safely in ClickHouse, progress can be off slightly
}
-
- console.log(`[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`);
} catch (error) {
console.error(`[Import ${importId}] ClickHouse insert failed:`, error);
diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.ts
index 9319348e0..b011ff622 100644
--- a/server/src/services/import/workers/jobs.ts
+++ b/server/src/services/import/workers/jobs.ts
@@ -20,7 +20,5 @@ export interface CsvParseJob extends ImportJob {
export interface DataInsertJob extends ImportJob {
chunk: UmamiEvent[];
- chunkNumber?: number; // Chunk sequence number for tracking
- totalChunks?: number; // Expected total chunks for finalization validation
allChunksSent: boolean; // Finalization signal
}