🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify data insert job
Edward Cui54c1d9fefd6c
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex fe1038d34..0736cd9d9 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -94,7 +94,6 @@ export async function registerCsvParseWorker() {let totalAccepted = 0;let totalSkippedQuota = 0;let totalSkippedDate = 0;- let chunksSent = 0;stream = isR2Storage? await createR2FileStream(storageLocation, platform)@@ -145,10 +144,8 @@ export async function registerCsvParseWorker() {importId,platform,chunk,- chunkNumber: chunksSent,allChunksSent: false,});- chunksSent++;chunk = [];}}@@ -180,10 +177,8 @@ export async function registerCsvParseWorker() {importId,platform,chunk,- chunkNumber: chunksSent,allChunksSent: false,});- chunksSent++;}// Send finalization signal with total chunk count@@ -192,7 +187,6 @@ export async function registerCsvParseWorker() {importId,platform,chunk: [],- totalChunks: chunksSent,allChunksSent: true,});} catch (error) {diff --git a/server/src/services/import/workers/dataInsertWorker.ts b/server/src/services/import/workers/dataInsertWorker.tsindex 407242af4..0ee20dbe7 100644--- a/server/src/services/import/workers/dataInsertWorker.ts+++ b/server/src/services/import/workers/dataInsertWorker.ts@@ -17,13 +17,11 @@ export async function registerDataInsertWorker() {const jobQueue = getJobQueue();await jobQueue.work<DataInsertJob>(DATA_INSERT_QUEUE, async job => {- const { site, importId, platform, chunk, chunkNumber, totalChunks, allChunksSent } = job;+ const { site, importId, platform, chunk, allChunksSent } = job;- // Handle finalization signalif (allChunksSent) {try {await updateImportStatus(importId, "completed");- console.log(`[Import ${importId}] Completed successfully (${totalChunks ?? 0} chunks processed)`);return;} catch (error) {console.error(`[Import ${importId}] Failed to mark as completed:`, error);@@ -38,7 +36,6 @@ export async function registerDataInsertWorker() {}}- // Process data chunktry {const dataMapper = getImportDataMapping(platform);const transformedRecords = dataMapper.transform(chunk, site, importId);@@ -60,8 +57,6 @@ export async function registerDataInsertWorker() {);// Don't throw - data is safely in ClickHouse, progress can be off slightly}-- console.log(`[Import ${importId}] Chunk ${chunkNumber ?? "?"} processed: ${transformedRecords.length} events`);} catch (error) {console.error(`[Import ${importId}] ClickHouse insert failed:`, error);diff --git a/server/src/services/import/workers/jobs.ts b/server/src/services/import/workers/jobs.tsindex 9319348e0..b011ff622 100644--- a/server/src/services/import/workers/jobs.ts+++ b/server/src/services/import/workers/jobs.ts@@ -20,7 +20,5 @@ export interface CsvParseJob extends ImportJob {export interface DataInsertJob extends ImportJob {chunk: UmamiEvent[];- chunkNumber?: number; // Chunk sequence number for tracking- totalChunks?: number; // Expected total chunks for finalization validationallChunksSent: boolean; // Finalization signal}