Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Queue

Edward Cui3c9c06c08dae
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index 5f0b9b800..934cfc12c 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -12,6 +12,10 @@ export class CSVWorkerManager {
private earliestAllowedDate: DateTime | null = null;
private latestAllowedDate: DateTime | null = null;
+ private uploadQueue: Array<UmamiEvent[]> = [];
+ private isProcessing = false;
+ private parsingComplete = false;
+
startImport(
file: File,
siteId: number,
@@ -38,15 +42,14 @@ export class CSVWorkerManager {
Papa.parse(file, {
header: true,
skipEmptyLines: "greedy",
+ worker: true,
chunkSize: 9 * 1024 * 1024,
- chunk: async (results, parser) => {
+ chunk: (results, parser) => {
if (this.aborted || this.quotaExceeded) {
parser.abort();
return;
}
- parser.pause();
-
const validEvents: UmamiEvent[] = [];
for (const row of results.data) {
const event = this.transformRow(row);
@@ -55,20 +58,14 @@ export class CSVWorkerManager {
}
}
- // Upload chunk and wait for completion
- await this.uploadChunk(validEvents, false);
-
- // Only resume if not aborted
- if (!this.aborted && !this.quotaExceeded) {
- parser.resume();
- }
+ this.uploadQueue.push(validEvents);
+ this.processQueue();
},
- complete: async () => {
+ complete: () => {
if (this.aborted) return;
- // Send finalization signal
- await this.uploadChunk([], true);
- console.log("Import completed successfully");
+ this.parsingComplete = true;
+ this.processQueue();
},
error: error => {
if (this.aborted) return;
@@ -77,6 +74,27 @@ export class CSVWorkerManager {
});
}
+ private async processQueue(): Promise<void> {
+ if (this.isProcessing) return;
+ this.isProcessing = true;
+
+ while (this.uploadQueue.length > 0) {
+ if (this.aborted || this.quotaExceeded) {
+ this.uploadQueue = [];
+ break;
+ }
+
+ const events = this.uploadQueue.shift()!;
+ await this.uploadChunk(events, false);
+ }
+
+ this.isProcessing = false;
+
+ if (this.parsingComplete && this.uploadQueue.length === 0 && !this.aborted) {
+ await this.uploadChunk([], true);
+ }
+ }
+
private isDateInRange(dateStr: string): boolean {
const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
if (!createdAt.isValid) {
@@ -128,7 +146,6 @@ export class CSVWorkerManager {
}
private handleError(message: string): void {
- console.error("CSV Import Error:", message);
this.terminate();
}
@@ -157,12 +174,10 @@ export class CSVWorkerManager {
if (data.quotaExceeded) {
this.quotaExceeded = true;
this.aborted = true;
- console.log("Import completed with quota limits:", data.message);
return;
}
} catch (error) {
// Critical failure - network error, server error, etc.
- console.error("Upload failed:", error instanceof Error ? error.message : "Unknown error");
this.terminate();
return;
}
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 77a9ee994..4d7689652 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -3,7 +3,7 @@ import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";
-import { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";
+import { UmamiImportMapper } from "../../services/import/mappings/umami.js";
import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
import { db } from "../../db/postgres/postgres.js";
import { sites, importStatus } from "../../db/postgres/schema.js";