Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Greatly simplify CSV parsing

Edward Cui46277ad2f2b2
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index 491b4a555..211f13ef9 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -1,14 +1,57 @@
-// Worker manager for coordinating CSV import process
-// Manages worker lifecycle and sequential batch uploads
+// CSV import manager using PapaParse with built-in worker
+// Handles streaming, batching, and sequential uploads
// Server handles transformation and quota checking
-import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, UmamiEvent } from "./types";
+import Papa from "papaparse";
+import { DateTime } from "luxon";
+import type { ImportProgress, UmamiEvent } from "./types";
type ProgressCallback = (progress: ImportProgress) => void;
type CompleteCallback = (success: boolean, message: string) => void;
+const BATCH_SIZE = 5000;
+
+const umamiHeaders = [
+ undefined,
+ "session_id",
+ undefined,
+ undefined,
+ "hostname",
+ "browser",
+ "os",
+ "device",
+ "screen",
+ "language",
+ "country",
+ "region",
+ "city",
+ "url_path",
+ "url_query",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ "referrer_path",
+ "referrer_query",
+ "referrer_domain",
+ "page_title",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ "event_type",
+ "event_name",
+ undefined,
+ "distinct_id",
+ "created_at",
+ undefined,
+];
+
export class CSVWorkerManager {
- private worker: Worker | null = null;
+ private aborted = false;
private progress: ImportProgress = {
status: "idle",
parsedRows: 0,
@@ -23,6 +66,10 @@ export class CSVWorkerManager {
private siteId: number = 0;
private importId: string = "";
+ private currentBatch: UmamiEvent[] = [];
+ private earliestAllowedDate: DateTime | null = null;
+ private latestAllowedDate: DateTime | null = null;
+
constructor(onProgress?: ProgressCallback, onComplete?: CompleteCallback) {
this.onProgress = onProgress || null;
this.onComplete = onComplete || null;
@@ -39,6 +86,8 @@ export class CSVWorkerManager {
this.importId = importId;
this.parsingComplete = false;
this.uploadInProgress = false;
+ this.currentBatch = [];
+ this.aborted = false;
this.progress = {
status: "parsing",
@@ -48,75 +97,145 @@ export class CSVWorkerManager {
errors: 0,
};
- // Create worker
- this.worker = new Worker(new URL("@/workers/csv-import.worker.ts", import.meta.url), {
- type: "module",
+ // Set up date range filter
+ this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
+ this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
+
+ if (!this.earliestAllowedDate.isValid) {
+ this.handleError(`Invalid earliest allowed date: ${earliestAllowedDate}`);
+ return;
+ }
+
+ if (!this.latestAllowedDate.isValid) {
+ this.handleError(`Invalid latest allowed date: ${latestAllowedDate}`);
+ return;
+ }
+
+ // Start parsing with PapaParse worker
+ Papa.parse<UmamiEvent>(file, {
+ worker: true,
+ header: true,
+ skipEmptyLines: "greedy",
+ delimiter: "",
+ transformHeader: (header, index) => {
+ return umamiHeaders[index] || header;
+ },
+ step: results => {
+ if (this.aborted) return;
+
+ if (results.data) {
+ this.handleParsedRow(results.data);
+ }
+ if (results.errors && results.errors.length > 0) {
+ this.progress.errors++;
+ }
+ },
+ complete: () => {
+ if (this.aborted) return;
+ this.handleParseComplete();
+ },
+ error: error => {
+ if (this.aborted) return;
+ this.handleError(error.message);
+ },
});
- // Set up message handler
- this.worker.onmessage = (event: MessageEvent<WorkerMessageToMain>) => {
- this.handleWorkerMessage(event.data);
- };
+ this.notifyProgress();
+ }
- // Set up error handler
- this.worker.onerror = error => {
- this.progress.status = "failed";
- this.progress.errors++;
- this.notifyProgress();
- if (this.onComplete) {
- this.onComplete(false, `Worker error: ${error.message}`);
- }
- };
+ private isDateInRange(dateStr: string): boolean {
+ const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ if (!createdAt.isValid) {
+ return false;
+ }
+
+ if (this.earliestAllowedDate && createdAt < this.earliestAllowedDate) {
+ return false;
+ }
+
+ if (this.latestAllowedDate && createdAt > this.latestAllowedDate) {
+ return false;
+ }
- // Start parsing with quota-based date range for client-side filtering
- const message: WorkerMessageToWorker = {
- type: "PARSE_START",
- file,
- earliestAllowedDate,
- latestAllowedDate,
+ return true;
+ }
+
+ private handleParsedRow(row: unknown): void {
+ const rawEvent = row as Record<string, unknown>;
+
+ const umamiEvent: UmamiEvent = {
+ session_id: String(rawEvent.session_id || ""),
+ hostname: String(rawEvent.hostname || ""),
+ browser: String(rawEvent.browser || ""),
+ os: String(rawEvent.os || ""),
+ device: String(rawEvent.device || ""),
+ screen: String(rawEvent.screen || ""),
+ language: String(rawEvent.language || ""),
+ country: String(rawEvent.country || ""),
+ region: String(rawEvent.region || ""),
+ city: String(rawEvent.city || ""),
+ url_path: String(rawEvent.url_path || ""),
+ url_query: String(rawEvent.url_query || ""),
+ referrer_path: String(rawEvent.referrer_path || ""),
+ referrer_query: String(rawEvent.referrer_query || ""),
+ referrer_domain: String(rawEvent.referrer_domain || ""),
+ page_title: String(rawEvent.page_title || ""),
+ event_type: String(rawEvent.event_type || ""),
+ event_name: String(rawEvent.event_name || ""),
+ distinct_id: String(rawEvent.distinct_id || ""),
+ created_at: String(rawEvent.created_at || ""),
};
- this.worker.postMessage(message);
- this.notifyProgress();
+ // Skip rows with missing created_at (required field)
+ if (!umamiEvent.created_at) {
+ this.progress.skippedRows++;
+ return;
+ }
+
+ // Apply quota-based date range filter
+ if (!this.isDateInRange(umamiEvent.created_at)) {
+ this.progress.skippedRows++;
+ return;
+ }
+
+ // Add to batch
+ this.currentBatch.push(umamiEvent);
+ this.progress.parsedRows++;
+
+ // Send batch when it reaches chunk size
+ if (this.currentBatch.length >= BATCH_SIZE) {
+ this.sendBatch();
+ }
}
- private handleWorkerMessage(message: WorkerMessageToMain): void {
- switch (message.type) {
- case "CHUNK_READY":
- // Update progress from chunk stats
- this.progress.parsedRows = message.parsed;
- this.progress.skippedRows = message.skipped;
- this.progress.errors = message.errors;
-
- // Upload batch immediately (sequential)
- this.progress.status = "uploading";
- this.notifyProgress();
- this.uploadBatch(message.events);
- break;
-
- case "COMPLETE":
- this.parsingComplete = true;
- this.progress.parsedRows = message.parsed;
- this.progress.skippedRows = message.skipped;
- this.progress.errors = message.errors;
-
- this.notifyProgress();
-
- // Check if upload is complete
- this.checkCompletion();
- break;
-
- case "ERROR":
- this.progress.status = "failed";
- this.progress.errors++;
- this.notifyProgress();
- if (this.onComplete) {
- this.onComplete(false, message.message);
- }
- break;
+ private sendBatch(): void {
+ if (this.currentBatch.length > 0) {
+ const batch = this.currentBatch;
+ this.currentBatch = [];
- default:
- console.warn("Unknown worker message type:", message);
+ // Upload batch (sequential)
+ this.progress.status = "uploading";
+ this.notifyProgress();
+ this.uploadBatch(batch);
+ }
+ }
+
+ private handleParseComplete(): void {
+ this.parsingComplete = true;
+
+ // Send final batch if any
+ this.sendBatch();
+
+ // Check if upload is complete
+ this.checkCompletion();
+ }
+
+ private handleError(message: string): void {
+ this.progress.status = "failed";
+ this.progress.errors++;
+ this.notifyProgress();
+ if (this.onComplete) {
+ this.onComplete(false, message);
}
}
@@ -170,9 +289,6 @@ export class CSVWorkerManager {
if (this.onComplete) {
this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);
}
-
- // Clean up worker
- this.terminate();
}
}
@@ -187,13 +303,6 @@ export class CSVWorkerManager {
}
terminate(): void {
- if (this.worker) {
- const message: WorkerMessageToWorker = {
- type: "CANCEL",
- };
- this.worker.postMessage(message);
- this.worker.terminate();
- this.worker = null;
- }
+ this.aborted = true;
}
}
diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.ts
index 724756111..94c98389a 100644
--- a/client/src/lib/import/types.ts
+++ b/client/src/lib/import/types.ts
@@ -26,38 +26,6 @@ export interface UmamiEvent {
created_at: string;
}
-// Worker message types
-export type WorkerMessageToWorker =
- | {
- type: "PARSE_START";
- file: File;
- earliestAllowedDate: string; // Filter events before this date (quota-based, yyyy-MM-dd)
- latestAllowedDate: string; // Filter events after this date (quota-based, yyyy-MM-dd)
- }
- | {
- type: "CANCEL";
- };
-
-export type WorkerMessageToMain =
- | {
- type: "CHUNK_READY";
- events: UmamiEvent[]; // Raw CSV rows, not transformed
- parsed: number; // Total rows parsed so far
- skipped: number; // Total rows skipped so far
- errors: number; // Total parse errors so far
- }
- | {
- type: "COMPLETE";
- parsed: number; // Total rows parsed
- skipped: number; // Total rows skipped
- errors: number; // Total parse errors
- }
- | {
- type: "ERROR";
- message: string;
- error?: unknown;
- };
-
// Batch import request (client sends raw rows to server)
export interface BatchImportRequest {
events: UmamiEvent[]; // Raw Umami CSV rows
diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.ts
deleted file mode 100644
index 626b112f9..000000000
--- a/client/src/workers/csv-import.worker.ts
+++ /dev/null
@@ -1,223 +0,0 @@
-import Papa from "papaparse";
-import { DateTime } from "luxon";
-import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";
-
-const BATCH_SIZE = 5000;
-
-let currentBatch: UmamiEvent[] = [];
-let totalParsed = 0;
-let totalSkipped = 0;
-let totalErrors = 0;
-
-let earliestAllowedDate: DateTime | null = null;
-let latestAllowedDate: DateTime | null = null;
-
-const umamiHeaders = [
- undefined,
- "session_id",
- undefined,
- undefined,
- "hostname",
- "browser",
- "os",
- "device",
- "screen",
- "language",
- "country",
- "region",
- "city",
- "url_path",
- "url_query",
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- "referrer_path",
- "referrer_query",
- "referrer_domain",
- "page_title",
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- "event_type",
- "event_name",
- undefined,
- "distinct_id",
- "created_at",
- undefined,
-];
-
-function createDateRangeFilter(earliestAllowedDateStr: string, latestAllowedDateStr: string) {
- earliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
- latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
-
- if (!earliestAllowedDate.isValid) {
- throw new Error(`Invalid earliest allowed date: ${earliestAllowedDateStr}`);
- }
-
- if (!latestAllowedDate.isValid) {
- throw new Error(`Invalid latest allowed date: ${latestAllowedDateStr}`);
- }
-}
-
-function isDateInRange(dateStr: string): boolean {
- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
- if (!createdAt.isValid) {
- return false;
- }
-
- if (earliestAllowedDate && createdAt < earliestAllowedDate) {
- return false;
- }
-
- if (latestAllowedDate && createdAt > latestAllowedDate) {
- return false;
- }
-
- return true;
-}
-
-function sendChunk() {
- if (currentBatch.length > 0) {
- const message: WorkerMessageToMain = {
- type: "CHUNK_READY",
- events: currentBatch,
- parsed: totalParsed,
- skipped: totalSkipped,
- errors: totalErrors,
- };
- self.postMessage(message);
- currentBatch = [];
- }
-}
-
-function handleParsedRow(row: unknown, rowIndex: number) {
- const rawEvent = row as Record<string, unknown>;
-
- // Filter out undefined columns (those mapped to undefined in umamiHeaders)
- // This reduces payload size by removing unused CSV columns
- const umamiEvent: UmamiEvent = {
- session_id: String(rawEvent.session_id || ""),
- hostname: String(rawEvent.hostname || ""),
- browser: String(rawEvent.browser || ""),
- os: String(rawEvent.os || ""),
- device: String(rawEvent.device || ""),
- screen: String(rawEvent.screen || ""),
- language: String(rawEvent.language || ""),
- country: String(rawEvent.country || ""),
- region: String(rawEvent.region || ""),
- city: String(rawEvent.city || ""),
- url_path: String(rawEvent.url_path || ""),
- url_query: String(rawEvent.url_query || ""),
- referrer_path: String(rawEvent.referrer_path || ""),
- referrer_query: String(rawEvent.referrer_query || ""),
- referrer_domain: String(rawEvent.referrer_domain || ""),
- page_title: String(rawEvent.page_title || ""),
- event_type: String(rawEvent.event_type || ""),
- event_name: String(rawEvent.event_name || ""),
- distinct_id: String(rawEvent.distinct_id || ""),
- created_at: String(rawEvent.created_at || ""),
- };
-
- // Skip rows with missing created_at (required field)
- if (!umamiEvent.created_at) {
- totalSkipped++;
- return;
- }
-
- // Apply quota-based date range filter
- // This client-side filtering reduces network traffic
- if (!isDateInRange(umamiEvent.created_at)) {
- totalSkipped++;
- return;
- }
-
- // Add to batch (filtered event with only required fields)
- currentBatch.push(umamiEvent);
- totalParsed++;
-
- // Send batch when it reaches chunk size
- if (currentBatch.length >= BATCH_SIZE) {
- sendChunk();
- }
-}
-
-function parseCSV(file: File) {
- let rowIndex = 0;
-
- Papa.parse<UmamiEvent>(file, {
- header: true,
- skipEmptyLines: "greedy", // Skip all empty lines (improved)
- delimiter: "", // Auto-detect delimiter (comma, tab, semicolon, etc.)
- transformHeader: (header, index) => {
- // Map Umami CSV column positions to field names
- return umamiHeaders[index] || header;
- },
- step: results => {
- if (results.data) {
- handleParsedRow(results.data, rowIndex);
- rowIndex++;
- }
- if (results.errors && results.errors.length > 0) {
- totalErrors++;
- }
- },
- complete: () => {
- // Send final chunk if any
- sendChunk();
-
- // Send completion message
- const message: WorkerMessageToMain = {
- type: "COMPLETE",
- parsed: totalParsed,
- skipped: totalSkipped,
- errors: totalErrors,
- };
- self.postMessage(message);
- },
- error: error => {
- const message: WorkerMessageToMain = {
- type: "ERROR",
- message: error.message,
- error,
- };
- self.postMessage(message);
- },
- });
-}
-
-// Listen for messages from main thread
-self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {
- const message = event.data;
-
- switch (message.type) {
- case "PARSE_START":
- // Reset state
- currentBatch = [];
- totalParsed = 0;
- totalSkipped = 0;
- totalErrors = 0;
-
- // Set up quota-based date range filter
- createDateRangeFilter(message.earliestAllowedDate, message.latestAllowedDate);
-
- // Start parsing
- parseCSV(message.file);
- break;
-
- case "CANCEL":
- // Terminate the worker
- self.close();
- break;
-
- default:
- console.warn("Unknown message type:", message);
- }
-};
-
-// Export empty object to make TypeScript happy
-export {};