🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Greatly simplify CSV parsing
Edward Cui46277ad2f2b2
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 491b4a555..211f13ef9 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -1,14 +1,57 @@-// Worker manager for coordinating CSV import process-// Manages worker lifecycle and sequential batch uploads+// CSV import manager using PapaParse with built-in worker+// Handles streaming, batching, and sequential uploads// Server handles transformation and quota checking-import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, UmamiEvent } from "./types";+import Papa from "papaparse";+import { DateTime } from "luxon";+import type { ImportProgress, UmamiEvent } from "./types";type ProgressCallback = (progress: ImportProgress) => void;type CompleteCallback = (success: boolean, message: string) => void;+const BATCH_SIZE = 5000;++const umamiHeaders = [+ undefined,+ "session_id",+ undefined,+ undefined,+ "hostname",+ "browser",+ "os",+ "device",+ "screen",+ "language",+ "country",+ "region",+ "city",+ "url_path",+ "url_query",+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ "referrer_path",+ "referrer_query",+ "referrer_domain",+ "page_title",+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ "event_type",+ "event_name",+ undefined,+ "distinct_id",+ "created_at",+ undefined,+];+export class CSVWorkerManager {- private worker: Worker | null = null;+ private aborted = false;private progress: ImportProgress = {status: "idle",parsedRows: 0,@@ -23,6 +66,10 @@ export class CSVWorkerManager {private siteId: number = 0;private importId: string = "";+ private currentBatch: UmamiEvent[] = [];+ private earliestAllowedDate: DateTime | null = null;+ private latestAllowedDate: DateTime | null = null;+constructor(onProgress?: ProgressCallback, onComplete?: CompleteCallback) {this.onProgress = onProgress || null;this.onComplete = onComplete || null;@@ -39,6 +86,8 @@ export class CSVWorkerManager {this.importId = importId;this.parsingComplete = false;this.uploadInProgress = false;+ this.currentBatch = [];+ this.aborted = false;this.progress = {status: "parsing",@@ -48,75 +97,145 @@ export class CSVWorkerManager {errors: 0,};- // Create worker- this.worker = new Worker(new URL("@/workers/csv-import.worker.ts", import.meta.url), {- type: "module",+ // Set up date range filter+ this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");+ this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");++ if (!this.earliestAllowedDate.isValid) {+ this.handleError(`Invalid earliest allowed date: ${earliestAllowedDate}`);+ return;+ }++ if (!this.latestAllowedDate.isValid) {+ this.handleError(`Invalid latest allowed date: ${latestAllowedDate}`);+ return;+ }++ // Start parsing with PapaParse worker+ Papa.parse<UmamiEvent>(file, {+ worker: true,+ header: true,+ skipEmptyLines: "greedy",+ delimiter: "",+ transformHeader: (header, index) => {+ return umamiHeaders[index] || header;+ },+ step: results => {+ if (this.aborted) return;++ if (results.data) {+ this.handleParsedRow(results.data);+ }+ if (results.errors && results.errors.length > 0) {+ this.progress.errors++;+ }+ },+ complete: () => {+ if (this.aborted) return;+ this.handleParseComplete();+ },+ error: error => {+ if (this.aborted) return;+ this.handleError(error.message);+ },});- // Set up message handler- this.worker.onmessage = (event: MessageEvent<WorkerMessageToMain>) => {- this.handleWorkerMessage(event.data);- };+ this.notifyProgress();+ }- // Set up error handler- this.worker.onerror = error => {- this.progress.status = "failed";- this.progress.errors++;- this.notifyProgress();- if (this.onComplete) {- this.onComplete(false, `Worker error: ${error.message}`);- }- };+ private isDateInRange(dateStr: string): boolean {+ const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ if (!createdAt.isValid) {+ return false;+ }++ if (this.earliestAllowedDate && createdAt < this.earliestAllowedDate) {+ return false;+ }++ if (this.latestAllowedDate && createdAt > this.latestAllowedDate) {+ return false;+ }- // Start parsing with quota-based date range for client-side filtering- const message: WorkerMessageToWorker = {- type: "PARSE_START",- file,- earliestAllowedDate,- latestAllowedDate,+ return true;+ }++ private handleParsedRow(row: unknown): void {+ const rawEvent = row as Record<string, unknown>;++ const umamiEvent: UmamiEvent = {+ session_id: String(rawEvent.session_id || ""),+ hostname: String(rawEvent.hostname || ""),+ browser: String(rawEvent.browser || ""),+ os: String(rawEvent.os || ""),+ device: String(rawEvent.device || ""),+ screen: String(rawEvent.screen || ""),+ language: String(rawEvent.language || ""),+ country: String(rawEvent.country || ""),+ region: String(rawEvent.region || ""),+ city: String(rawEvent.city || ""),+ url_path: String(rawEvent.url_path || ""),+ url_query: String(rawEvent.url_query || ""),+ referrer_path: String(rawEvent.referrer_path || ""),+ referrer_query: String(rawEvent.referrer_query || ""),+ referrer_domain: String(rawEvent.referrer_domain || ""),+ page_title: String(rawEvent.page_title || ""),+ event_type: String(rawEvent.event_type || ""),+ event_name: String(rawEvent.event_name || ""),+ distinct_id: String(rawEvent.distinct_id || ""),+ created_at: String(rawEvent.created_at || ""),};- this.worker.postMessage(message);- this.notifyProgress();+ // Skip rows with missing created_at (required field)+ if (!umamiEvent.created_at) {+ this.progress.skippedRows++;+ return;+ }++ // Apply quota-based date range filter+ if (!this.isDateInRange(umamiEvent.created_at)) {+ this.progress.skippedRows++;+ return;+ }++ // Add to batch+ this.currentBatch.push(umamiEvent);+ this.progress.parsedRows++;++ // Send batch when it reaches chunk size+ if (this.currentBatch.length >= BATCH_SIZE) {+ this.sendBatch();+ }}- private handleWorkerMessage(message: WorkerMessageToMain): void {- switch (message.type) {- case "CHUNK_READY":- // Update progress from chunk stats- this.progress.parsedRows = message.parsed;- this.progress.skippedRows = message.skipped;- this.progress.errors = message.errors;-- // Upload batch immediately (sequential)- this.progress.status = "uploading";- this.notifyProgress();- this.uploadBatch(message.events);- break;-- case "COMPLETE":- this.parsingComplete = true;- this.progress.parsedRows = message.parsed;- this.progress.skippedRows = message.skipped;- this.progress.errors = message.errors;-- this.notifyProgress();-- // Check if upload is complete- this.checkCompletion();- break;-- case "ERROR":- this.progress.status = "failed";- this.progress.errors++;- this.notifyProgress();- if (this.onComplete) {- this.onComplete(false, message.message);- }- break;+ private sendBatch(): void {+ if (this.currentBatch.length > 0) {+ const batch = this.currentBatch;+ this.currentBatch = [];- default:- console.warn("Unknown worker message type:", message);+ // Upload batch (sequential)+ this.progress.status = "uploading";+ this.notifyProgress();+ this.uploadBatch(batch);+ }+ }++ private handleParseComplete(): void {+ this.parsingComplete = true;++ // Send final batch if any+ this.sendBatch();++ // Check if upload is complete+ this.checkCompletion();+ }++ private handleError(message: string): void {+ this.progress.status = "failed";+ this.progress.errors++;+ this.notifyProgress();+ if (this.onComplete) {+ this.onComplete(false, message);}}@@ -170,9 +289,6 @@ export class CSVWorkerManager {if (this.onComplete) {this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);}-- // Clean up worker- this.terminate();}}@@ -187,13 +303,6 @@ export class CSVWorkerManager {}terminate(): void {- if (this.worker) {- const message: WorkerMessageToWorker = {- type: "CANCEL",- };- this.worker.postMessage(message);- this.worker.terminate();- this.worker = null;- }+ this.aborted = true;}}diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex 724756111..94c98389a 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -26,38 +26,6 @@ export interface UmamiEvent {created_at: string;}-// Worker message types-export type WorkerMessageToWorker =- | {- type: "PARSE_START";- file: File;- earliestAllowedDate: string; // Filter events before this date (quota-based, yyyy-MM-dd)- latestAllowedDate: string; // Filter events after this date (quota-based, yyyy-MM-dd)- }- | {- type: "CANCEL";- };--export type WorkerMessageToMain =- | {- type: "CHUNK_READY";- events: UmamiEvent[]; // Raw CSV rows, not transformed- parsed: number; // Total rows parsed so far- skipped: number; // Total rows skipped so far- errors: number; // Total parse errors so far- }- | {- type: "COMPLETE";- parsed: number; // Total rows parsed- skipped: number; // Total rows skipped- errors: number; // Total parse errors- }- | {- type: "ERROR";- message: string;- error?: unknown;- };-// Batch import request (client sends raw rows to server)export interface BatchImportRequest {events: UmamiEvent[]; // Raw Umami CSV rowsdiff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsdeleted file mode 100644index 626b112f9..000000000--- a/client/src/workers/csv-import.worker.ts+++ /dev/null@@ -1,223 +0,0 @@-import Papa from "papaparse";-import { DateTime } from "luxon";-import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";--const BATCH_SIZE = 5000;--let currentBatch: UmamiEvent[] = [];-let totalParsed = 0;-let totalSkipped = 0;-let totalErrors = 0;--let earliestAllowedDate: DateTime | null = null;-let latestAllowedDate: DateTime | null = null;--const umamiHeaders = [- undefined,- "session_id",- undefined,- undefined,- "hostname",- "browser",- "os",- "device",- "screen",- "language",- "country",- "region",- "city",- "url_path",- "url_query",- undefined,- undefined,- undefined,- undefined,- undefined,- "referrer_path",- "referrer_query",- "referrer_domain",- "page_title",- undefined,- undefined,- undefined,- undefined,- undefined,- undefined,- "event_type",- "event_name",- undefined,- "distinct_id",- "created_at",- undefined,-];--function createDateRangeFilter(earliestAllowedDateStr: string, latestAllowedDateStr: string) {- earliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");- latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");-- if (!earliestAllowedDate.isValid) {- throw new Error(`Invalid earliest allowed date: ${earliestAllowedDateStr}`);- }-- if (!latestAllowedDate.isValid) {- throw new Error(`Invalid latest allowed date: ${latestAllowedDateStr}`);- }-}--function isDateInRange(dateStr: string): boolean {- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });- if (!createdAt.isValid) {- return false;- }-- if (earliestAllowedDate && createdAt < earliestAllowedDate) {- return false;- }-- if (latestAllowedDate && createdAt > latestAllowedDate) {- return false;- }-- return true;-}--function sendChunk() {- if (currentBatch.length > 0) {- const message: WorkerMessageToMain = {- type: "CHUNK_READY",- events: currentBatch,- parsed: totalParsed,- skipped: totalSkipped,- errors: totalErrors,- };- self.postMessage(message);- currentBatch = [];- }-}--function handleParsedRow(row: unknown, rowIndex: number) {- const rawEvent = row as Record<string, unknown>;-- // Filter out undefined columns (those mapped to undefined in umamiHeaders)- // This reduces payload size by removing unused CSV columns- const umamiEvent: UmamiEvent = {- session_id: String(rawEvent.session_id || ""),- hostname: String(rawEvent.hostname || ""),- browser: String(rawEvent.browser || ""),- os: String(rawEvent.os || ""),- device: String(rawEvent.device || ""),- screen: String(rawEvent.screen || ""),- language: String(rawEvent.language || ""),- country: String(rawEvent.country || ""),- region: String(rawEvent.region || ""),- city: String(rawEvent.city || ""),- url_path: String(rawEvent.url_path || ""),- url_query: String(rawEvent.url_query || ""),- referrer_path: String(rawEvent.referrer_path || ""),- referrer_query: String(rawEvent.referrer_query || ""),- referrer_domain: String(rawEvent.referrer_domain || ""),- page_title: String(rawEvent.page_title || ""),- event_type: String(rawEvent.event_type || ""),- event_name: String(rawEvent.event_name || ""),- distinct_id: String(rawEvent.distinct_id || ""),- created_at: String(rawEvent.created_at || ""),- };-- // Skip rows with missing created_at (required field)- if (!umamiEvent.created_at) {- totalSkipped++;- return;- }-- // Apply quota-based date range filter- // This client-side filtering reduces network traffic- if (!isDateInRange(umamiEvent.created_at)) {- totalSkipped++;- return;- }-- // Add to batch (filtered event with only required fields)- currentBatch.push(umamiEvent);- totalParsed++;-- // Send batch when it reaches chunk size- if (currentBatch.length >= BATCH_SIZE) {- sendChunk();- }-}--function parseCSV(file: File) {- let rowIndex = 0;-- Papa.parse<UmamiEvent>(file, {- header: true,- skipEmptyLines: "greedy", // Skip all empty lines (improved)- delimiter: "", // Auto-detect delimiter (comma, tab, semicolon, etc.)- transformHeader: (header, index) => {- // Map Umami CSV column positions to field names- return umamiHeaders[index] || header;- },- step: results => {- if (results.data) {- handleParsedRow(results.data, rowIndex);- rowIndex++;- }- if (results.errors && results.errors.length > 0) {- totalErrors++;- }- },- complete: () => {- // Send final chunk if any- sendChunk();-- // Send completion message- const message: WorkerMessageToMain = {- type: "COMPLETE",- parsed: totalParsed,- skipped: totalSkipped,- errors: totalErrors,- };- self.postMessage(message);- },- error: error => {- const message: WorkerMessageToMain = {- type: "ERROR",- message: error.message,- error,- };- self.postMessage(message);- },- });-}--// Listen for messages from main thread-self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {- const message = event.data;-- switch (message.type) {- case "PARSE_START":- // Reset state- currentBatch = [];- totalParsed = 0;- totalSkipped = 0;- totalErrors = 0;-- // Set up quota-based date range filter- createDateRangeFilter(message.earliestAllowedDate, message.latestAllowedDate);-- // Start parsing- parseCSV(message.file);- break;-- case "CANCEL":- // Terminate the worker- self.close();- break;-- default:- console.warn("Unknown message type:", message);- }-};--// Export empty object to make TypeScript happy-export {};