🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Queue
Edward Cui3c9c06c08dae
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 5f0b9b800..934cfc12c 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -12,6 +12,10 @@ export class CSVWorkerManager {private earliestAllowedDate: DateTime | null = null;private latestAllowedDate: DateTime | null = null;+ private uploadQueue: Array<UmamiEvent[]> = [];+ private isProcessing = false;+ private parsingComplete = false;+startImport(file: File,siteId: number,@@ -38,15 +42,14 @@ export class CSVWorkerManager {Papa.parse(file, {header: true,skipEmptyLines: "greedy",+ worker: true,chunkSize: 9 * 1024 * 1024,- chunk: async (results, parser) => {+ chunk: (results, parser) => {if (this.aborted || this.quotaExceeded) {parser.abort();return;}- parser.pause();-const validEvents: UmamiEvent[] = [];for (const row of results.data) {const event = this.transformRow(row);@@ -55,20 +58,14 @@ export class CSVWorkerManager {}}- // Upload chunk and wait for completion- await this.uploadChunk(validEvents, false);-- // Only resume if not aborted- if (!this.aborted && !this.quotaExceeded) {- parser.resume();- }+ this.uploadQueue.push(validEvents);+ this.processQueue();},- complete: async () => {+ complete: () => {if (this.aborted) return;- // Send finalization signal- await this.uploadChunk([], true);- console.log("Import completed successfully");+ this.parsingComplete = true;+ this.processQueue();},error: error => {if (this.aborted) return;@@ -77,6 +74,27 @@ export class CSVWorkerManager {});}+ private async processQueue(): Promise<void> {+ if (this.isProcessing) return;+ this.isProcessing = true;++ while (this.uploadQueue.length > 0) {+ if (this.aborted || this.quotaExceeded) {+ this.uploadQueue = [];+ break;+ }++ const events = this.uploadQueue.shift()!;+ await this.uploadChunk(events, false);+ }++ this.isProcessing = false;++ if (this.parsingComplete && this.uploadQueue.length === 0 && !this.aborted) {+ await this.uploadChunk([], true);+ }+ }+private isDateInRange(dateStr: string): boolean {const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });if (!createdAt.isValid) {@@ -128,7 +146,6 @@ export class CSVWorkerManager {}private handleError(message: string): void {- console.error("CSV Import Error:", message);this.terminate();}@@ -157,12 +174,10 @@ export class CSVWorkerManager {if (data.quotaExceeded) {this.quotaExceeded = true;this.aborted = true;- console.log("Import completed with quota limits:", data.message);return;}} catch (error) {// Critical failure - network error, server error, etc.- console.error("Upload failed:", error instanceof Error ? error.message : "Unknown error");this.terminate();return;}diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 77a9ee994..4d7689652 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -3,7 +3,7 @@ import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";-import { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";+import { UmamiImportMapper } from "../../services/import/mappings/umami.js";import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";import { db } from "../../db/postgres/postgres.js";import { sites, importStatus } from "../../db/postgres/schema.js";