🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify CSV parser
Edward Cuibb43267205d3
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex c1d3587b3..f6637869e 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -524,17 +524,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</div>{/* Errors */}- {importProgress && importProgress.errors.length > 0 && (- <div className="max-h-40 overflow-y-auto">- <p className="text-sm font-medium text-red-600 mb-2">Errors ({importProgress.errors.length}):</p>- {importProgress.errors.slice(0, 10).map((error, idx) => (- <p key={idx} className="text-xs text-red-600">- {error.message}- </p>- ))}- {importProgress.errors.length > 10 && (- <p className="text-xs text-muted-foreground">...and {importProgress.errors.length - 10} more</p>- )}+ {importProgress && importProgress.errors > 0 && (+ <div>+ <p className="text-sm font-medium text-red-600">{importProgress.errors} parse errors occurred</p></div>)}</div>diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex d57778ffd..491b4a555 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -11,11 +11,10 @@ export class CSVWorkerManager {private worker: Worker | null = null;private progress: ImportProgress = {status: "idle",- totalRows: 0,parsedRows: 0,skippedRows: 0,importedEvents: 0,- errors: [],+ errors: 0,};private onProgress: ProgressCallback | null = null;private onComplete: CompleteCallback | null = null;@@ -43,11 +42,10 @@ export class CSVWorkerManager {this.progress = {status: "parsing",- totalRows: 0,parsedRows: 0,skippedRows: 0,importedEvents: 0,- errors: [],+ errors: 0,};// Create worker@@ -63,9 +61,7 @@ export class CSVWorkerManager {// Set up error handlerthis.worker.onerror = error => {this.progress.status = "failed";- this.progress.errors.push({- message: `Worker error: ${error.message}`,- });+ this.progress.errors++;this.notifyProgress();if (this.onComplete) {this.onComplete(false, `Worker error: ${error.message}`);@@ -76,8 +72,6 @@ export class CSVWorkerManager {const message: WorkerMessageToWorker = {type: "PARSE_START",file,- siteId,- importId,earliestAllowedDate,latestAllowedDate,};@@ -88,17 +82,12 @@ export class CSVWorkerManager {private handleWorkerMessage(message: WorkerMessageToMain): void {switch (message.type) {- case "PROGRESS":+ case "CHUNK_READY":+ // Update progress from chunk statsthis.progress.parsedRows = message.parsed;this.progress.skippedRows = message.skipped;- this.progress.errors = this.progress.errors.slice(0, this.progress.errors.length - message.errors);- for (let i = 0; i < message.errors; i++) {- this.progress.errors.push({ message: "Parse error" });- }- this.notifyProgress();- break;+ this.progress.errors = message.errors;- case "CHUNK_READY":// Upload batch immediately (sequential)this.progress.status = "uploading";this.notifyProgress();@@ -107,15 +96,9 @@ export class CSVWorkerManager {case "COMPLETE":this.parsingComplete = true;- this.progress.parsedRows = message.totalParsed;- this.progress.skippedRows = message.totalSkipped;-- // Add detailed errors- message.errorDetails.forEach(error => {- this.progress.errors.push({- message: `Row ${error.row}: ${error.message}`,- });- });+ this.progress.parsedRows = message.parsed;+ this.progress.skippedRows = message.skipped;+ this.progress.errors = message.errors;this.notifyProgress();@@ -125,9 +108,7 @@ export class CSVWorkerManager {case "ERROR":this.progress.status = "failed";- this.progress.errors.push({- message: message.message,- });+ this.progress.errors++;this.notifyProgress();if (this.onComplete) {this.onComplete(false, message.message);@@ -165,9 +146,7 @@ export class CSVWorkerManager {} catch (error) {// Fail fast - no retriesthis.progress.status = "failed";- this.progress.errors.push({- message: `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`,- });+ this.progress.errors++;this.notifyProgress();if (this.onComplete) {diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex d0837df4b..724756111 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -31,8 +31,6 @@ export type WorkerMessageToWorker =| {type: "PARSE_START";file: File;- siteId: number;- importId: string;earliestAllowedDate: string; // Filter events before this date (quota-based, yyyy-MM-dd)latestAllowedDate: string; // Filter events after this date (quota-based, yyyy-MM-dd)}@@ -41,22 +39,18 @@ export type WorkerMessageToWorker =};export type WorkerMessageToMain =- | {- type: "PROGRESS";- parsed: number;- skipped: number;- errors: number;- }| {type: "CHUNK_READY";events: UmamiEvent[]; // Raw CSV rows, not transformed+ parsed: number; // Total rows parsed so far+ skipped: number; // Total rows skipped so far+ errors: number; // Total parse errors so far}| {type: "COMPLETE";- totalParsed: number;- totalSkipped: number;- totalErrors: number;- errorDetails: Array<{ row: number; message: string }>;+ parsed: number; // Total rows parsed+ skipped: number; // Total rows skipped+ errors: number; // Total parse errors}| {type: "ERROR";@@ -81,9 +75,8 @@ export interface BatchImportResponse {// Import progress trackingexport interface ImportProgress {status: "idle" | "parsing" | "uploading" | "completed" | "failed";- totalRows: number;parsedRows: number;skippedRows: number;importedEvents: number;- errors: Array<{ message: string }>;+ errors: number;}diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsindex 4b2ffcb6c..626b112f9 100644--- a/client/src/workers/csv-import.worker.ts+++ b/client/src/workers/csv-import.worker.ts@@ -1,27 +1,17 @@-// Web Worker for client-side CSV parsing only-// This worker runs in a separate thread to avoid blocking the main UI thread-// Transformation happens on the server-import Papa from "papaparse";import { DateTime } from "luxon";import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";-const CHUNK_SIZE = 5000; // Number of rows per batch sent to main thread-const PROGRESS_UPDATE_INTERVAL = 1000; // Update progress every 1000 rows+const BATCH_SIZE = 5000;let currentBatch: UmamiEvent[] = [];let totalParsed = 0;let totalSkipped = 0;let totalErrors = 0;-let errorDetails: Array<{ row: number; message: string }> = [];-let lastProgressUpdate = 0;-// Date range filters (client-side optimization to reduce server load)-// Quota-enforced dates (based on subscription tier)let earliestAllowedDate: DateTime | null = null;let latestAllowedDate: DateTime | null = null;-// Umami CSV header mappingconst umamiHeaders = [undefined,"session_id",@@ -62,7 +52,6 @@ const umamiHeaders = [];function createDateRangeFilter(earliestAllowedDateStr: string, latestAllowedDateStr: string) {- // Parse quota-enforced datesearliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");@@ -81,7 +70,6 @@ function isDateInRange(dateStr: string): boolean {return false;}- // Check quota-enforced datesif (earliestAllowedDate && createdAt < earliestAllowedDate) {return false;}@@ -98,22 +86,15 @@ function sendChunk() {const message: WorkerMessageToMain = {type: "CHUNK_READY",events: currentBatch,+ parsed: totalParsed,+ skipped: totalSkipped,+ errors: totalErrors,};self.postMessage(message);currentBatch = [];}}-function sendProgress() {- const message: WorkerMessageToMain = {- type: "PROGRESS",- parsed: totalParsed,- skipped: totalSkipped,- errors: totalErrors,- };- self.postMessage(message);-}-function handleParsedRow(row: unknown, rowIndex: number) {const rawEvent = row as Record<string, unknown>;@@ -160,15 +141,9 @@ function handleParsedRow(row: unknown, rowIndex: number) {totalParsed++;// Send batch when it reaches chunk size- if (currentBatch.length >= CHUNK_SIZE) {+ if (currentBatch.length >= BATCH_SIZE) {sendChunk();}-- // Send progress update periodically- if (totalParsed - lastProgressUpdate >= PROGRESS_UPDATE_INTERVAL) {- sendProgress();- lastProgressUpdate = totalParsed;- }}function parseCSV(file: File) {@@ -189,29 +164,18 @@ function parseCSV(file: File) {}if (results.errors && results.errors.length > 0) {totalErrors++;- if (errorDetails.length < 100) {- // Limit error collection to avoid memory issues- errorDetails.push({- row: rowIndex,- message: results.errors.map(e => e.message).join(", "),- });- }}},complete: () => {// Send final chunk if anysendChunk();- // Send final progress- sendProgress();-// Send completion messageconst message: WorkerMessageToMain = {type: "COMPLETE",- totalParsed,- totalSkipped,- totalErrors,- errorDetails,+ parsed: totalParsed,+ skipped: totalSkipped,+ errors: totalErrors,};self.postMessage(message);},@@ -237,8 +201,6 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {totalParsed = 0;totalSkipped = 0;totalErrors = 0;- errorDetails = [];- lastProgressUpdate = 0;// Set up quota-based date range filtercreateDateRangeFilter(message.earliestAllowedDate, message.latestAllowedDate);