🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify csv worker
Edward Cui45555cdf71ea
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.tsindex 0736cd9d9..8953eb07f 100644--- a/server/src/services/import/workers/csvParseWorker.ts+++ b/server/src/services/import/workers/csvParseWorker.ts@@ -91,9 +91,7 @@ export async function registerCsvParseWorker() {const PROCESSING_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes maxlet chunk: UmamiEvent[] = [];- let totalAccepted = 0;let totalSkippedQuota = 0;- let totalSkippedDate = 0;stream = isR2Storage? await createR2FileStream(storageLocation, platform)@@ -124,7 +122,6 @@ export async function registerCsvParseWorker() {// Apply user-specified date range filterif (!isDateInRange(data.created_at)) {- totalSkippedDate++;continue;}@@ -136,7 +133,6 @@ export async function registerCsvParseWorker() {// Event passed all filters - add to chunkchunk.push(data);- totalAccepted++;if (chunk.length >= chunkSize) {await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {@@ -150,16 +146,22 @@ export async function registerCsvParseWorker() {}}- console.info(- `[Import ${importId}] Processed CSV: ${totalAccepted} events accepted, ` +- `${totalSkippedQuota} skipped (quota/window), ${totalSkippedDate} skipped (date filter)`- );+ // Send final chunk if any data remains+ if (chunk.length > 0) {+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {+ site,+ importId,+ platform,+ chunk,+ allChunksSent: false,+ });+ }- // Check if no events could be imported due to quotas- if (totalAccepted === 0 && totalSkippedQuota > 0) {+ // Check if some events couldn't be imported due to quotas+ if (totalSkippedQuota > 0) {const quotaSummary = quotaTracker.getSummary();const errorMessage =- `No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` ++ `${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +`Try importing newer data or upgrade your plan for higher monthly quotas.`;await updateImportStatus(importId, "failed", errorMessage);@@ -170,17 +172,6 @@ export async function registerCsvParseWorker() {return;}- // Send final chunk if any data remains- if (chunk.length > 0) {- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {- site,- importId,- platform,- chunk,- allChunksSent: false,- });- }-// Send finalization signal with total chunk countawait jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {site,