Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify csv worker

Edward Cui45555cdf71ea
diff --git a/server/src/services/import/workers/csvParseWorker.ts b/server/src/services/import/workers/csvParseWorker.ts
index 0736cd9d9..8953eb07f 100644
--- a/server/src/services/import/workers/csvParseWorker.ts
+++ b/server/src/services/import/workers/csvParseWorker.ts
@@ -91,9 +91,7 @@ export async function registerCsvParseWorker() {
const PROCESSING_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes max
let chunk: UmamiEvent[] = [];
- let totalAccepted = 0;
let totalSkippedQuota = 0;
- let totalSkippedDate = 0;
stream = isR2Storage
? await createR2FileStream(storageLocation, platform)
@@ -124,7 +122,6 @@ export async function registerCsvParseWorker() {
// Apply user-specified date range filter
if (!isDateInRange(data.created_at)) {
- totalSkippedDate++;
continue;
}
@@ -136,7 +133,6 @@ export async function registerCsvParseWorker() {
// Event passed all filters - add to chunk
chunk.push(data);
- totalAccepted++;
if (chunk.length >= chunkSize) {
await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
@@ -150,16 +146,22 @@ export async function registerCsvParseWorker() {
}
}
- console.info(
- `[Import ${importId}] Processed CSV: ${totalAccepted} events accepted, ` +
- `${totalSkippedQuota} skipped (quota/window), ${totalSkippedDate} skipped (date filter)`
- );
+ // Send final chunk if any data remains
+ if (chunk.length > 0) {
+ await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
+ site,
+ importId,
+ platform,
+ chunk,
+ allChunksSent: false,
+ });
+ }
- // Check if no events could be imported due to quotas
- if (totalAccepted === 0 && totalSkippedQuota > 0) {
+ // Check if some events couldn't be imported due to quotas
+ if (totalSkippedQuota > 0) {
const quotaSummary = quotaTracker.getSummary();
const errorMessage =
- `No events could be imported. All ${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
+ `${totalSkippedQuota} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +
`Try importing newer data or upgrade your plan for higher monthly quotas.`;
await updateImportStatus(importId, "failed", errorMessage);
@@ -170,17 +172,6 @@ export async function registerCsvParseWorker() {
return;
}
- // Send final chunk if any data remains
- if (chunk.length > 0) {
- await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
- site,
- importId,
- platform,
- chunk,
- allChunksSent: false,
- });
- }
-
// Send finalization signal with total chunk count
await jobQueue.send<DataInsertJob>(DATA_INSERT_QUEUE, {
site,