Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify CSV parser

Edward Cuibb43267205d3
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index c1d3587b3..f6637869e 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -524,17 +524,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</div>
{/* Errors */}
- {importProgress && importProgress.errors.length > 0 && (
- <div className="max-h-40 overflow-y-auto">
- <p className="text-sm font-medium text-red-600 mb-2">Errors ({importProgress.errors.length}):</p>
- {importProgress.errors.slice(0, 10).map((error, idx) => (
- <p key={idx} className="text-xs text-red-600">
- {error.message}
- </p>
- ))}
- {importProgress.errors.length > 10 && (
- <p className="text-xs text-muted-foreground">...and {importProgress.errors.length - 10} more</p>
- )}
+ {importProgress && importProgress.errors > 0 && (
+ <div>
+ <p className="text-sm font-medium text-red-600">{importProgress.errors} parse errors occurred</p>
</div>
)}
</div>
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index d57778ffd..491b4a555 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -11,11 +11,10 @@ export class CSVWorkerManager {
private worker: Worker | null = null;
private progress: ImportProgress = {
status: "idle",
- totalRows: 0,
parsedRows: 0,
skippedRows: 0,
importedEvents: 0,
- errors: [],
+ errors: 0,
};
private onProgress: ProgressCallback | null = null;
private onComplete: CompleteCallback | null = null;
@@ -43,11 +42,10 @@ export class CSVWorkerManager {
this.progress = {
status: "parsing",
- totalRows: 0,
parsedRows: 0,
skippedRows: 0,
importedEvents: 0,
- errors: [],
+ errors: 0,
};
// Create worker
@@ -63,9 +61,7 @@ export class CSVWorkerManager {
// Set up error handler
this.worker.onerror = error => {
this.progress.status = "failed";
- this.progress.errors.push({
- message: `Worker error: ${error.message}`,
- });
+ this.progress.errors++;
this.notifyProgress();
if (this.onComplete) {
this.onComplete(false, `Worker error: ${error.message}`);
@@ -76,8 +72,6 @@ export class CSVWorkerManager {
const message: WorkerMessageToWorker = {
type: "PARSE_START",
file,
- siteId,
- importId,
earliestAllowedDate,
latestAllowedDate,
};
@@ -88,17 +82,12 @@ export class CSVWorkerManager {
private handleWorkerMessage(message: WorkerMessageToMain): void {
switch (message.type) {
- case "PROGRESS":
+ case "CHUNK_READY":
+ // Update progress from chunk stats
this.progress.parsedRows = message.parsed;
this.progress.skippedRows = message.skipped;
- this.progress.errors = this.progress.errors.slice(0, this.progress.errors.length - message.errors);
- for (let i = 0; i < message.errors; i++) {
- this.progress.errors.push({ message: "Parse error" });
- }
- this.notifyProgress();
- break;
+ this.progress.errors = message.errors;
- case "CHUNK_READY":
// Upload batch immediately (sequential)
this.progress.status = "uploading";
this.notifyProgress();
@@ -107,15 +96,9 @@ export class CSVWorkerManager {
case "COMPLETE":
this.parsingComplete = true;
- this.progress.parsedRows = message.totalParsed;
- this.progress.skippedRows = message.totalSkipped;
-
- // Add detailed errors
- message.errorDetails.forEach(error => {
- this.progress.errors.push({
- message: `Row ${error.row}: ${error.message}`,
- });
- });
+ this.progress.parsedRows = message.parsed;
+ this.progress.skippedRows = message.skipped;
+ this.progress.errors = message.errors;
this.notifyProgress();
@@ -125,9 +108,7 @@ export class CSVWorkerManager {
case "ERROR":
this.progress.status = "failed";
- this.progress.errors.push({
- message: message.message,
- });
+ this.progress.errors++;
this.notifyProgress();
if (this.onComplete) {
this.onComplete(false, message.message);
@@ -165,9 +146,7 @@ export class CSVWorkerManager {
} catch (error) {
// Fail fast - no retries
this.progress.status = "failed";
- this.progress.errors.push({
- message: `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`,
- });
+ this.progress.errors++;
this.notifyProgress();
if (this.onComplete) {
diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.ts
index d0837df4b..724756111 100644
--- a/client/src/lib/import/types.ts
+++ b/client/src/lib/import/types.ts
@@ -31,8 +31,6 @@ export type WorkerMessageToWorker =
| {
type: "PARSE_START";
file: File;
- siteId: number;
- importId: string;
earliestAllowedDate: string; // Filter events before this date (quota-based, yyyy-MM-dd)
latestAllowedDate: string; // Filter events after this date (quota-based, yyyy-MM-dd)
}
@@ -41,22 +39,18 @@ export type WorkerMessageToWorker =
};
export type WorkerMessageToMain =
- | {
- type: "PROGRESS";
- parsed: number;
- skipped: number;
- errors: number;
- }
| {
type: "CHUNK_READY";
events: UmamiEvent[]; // Raw CSV rows, not transformed
+ parsed: number; // Total rows parsed so far
+ skipped: number; // Total rows skipped so far
+ errors: number; // Total parse errors so far
}
| {
type: "COMPLETE";
- totalParsed: number;
- totalSkipped: number;
- totalErrors: number;
- errorDetails: Array<{ row: number; message: string }>;
+ parsed: number; // Total rows parsed
+ skipped: number; // Total rows skipped
+ errors: number; // Total parse errors
}
| {
type: "ERROR";
@@ -81,9 +75,8 @@ export interface BatchImportResponse {
// Import progress tracking
export interface ImportProgress {
status: "idle" | "parsing" | "uploading" | "completed" | "failed";
- totalRows: number;
parsedRows: number;
skippedRows: number;
importedEvents: number;
- errors: Array<{ message: string }>;
+ errors: number;
}
diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.ts
index 4b2ffcb6c..626b112f9 100644
--- a/client/src/workers/csv-import.worker.ts
+++ b/client/src/workers/csv-import.worker.ts
@@ -1,27 +1,17 @@
-// Web Worker for client-side CSV parsing only
-// This worker runs in a separate thread to avoid blocking the main UI thread
-// Transformation happens on the server
-
import Papa from "papaparse";
import { DateTime } from "luxon";
import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";
-const CHUNK_SIZE = 5000; // Number of rows per batch sent to main thread
-const PROGRESS_UPDATE_INTERVAL = 1000; // Update progress every 1000 rows
+const BATCH_SIZE = 5000;
let currentBatch: UmamiEvent[] = [];
let totalParsed = 0;
let totalSkipped = 0;
let totalErrors = 0;
-let errorDetails: Array<{ row: number; message: string }> = [];
-let lastProgressUpdate = 0;
-// Date range filters (client-side optimization to reduce server load)
-// Quota-enforced dates (based on subscription tier)
let earliestAllowedDate: DateTime | null = null;
let latestAllowedDate: DateTime | null = null;
-// Umami CSV header mapping
const umamiHeaders = [
undefined,
"session_id",
@@ -62,7 +52,6 @@ const umamiHeaders = [
];
function createDateRangeFilter(earliestAllowedDateStr: string, latestAllowedDateStr: string) {
- // Parse quota-enforced dates
earliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
@@ -81,7 +70,6 @@ function isDateInRange(dateStr: string): boolean {
return false;
}
- // Check quota-enforced dates
if (earliestAllowedDate && createdAt < earliestAllowedDate) {
return false;
}
@@ -98,22 +86,15 @@ function sendChunk() {
const message: WorkerMessageToMain = {
type: "CHUNK_READY",
events: currentBatch,
+ parsed: totalParsed,
+ skipped: totalSkipped,
+ errors: totalErrors,
};
self.postMessage(message);
currentBatch = [];
}
}
-function sendProgress() {
- const message: WorkerMessageToMain = {
- type: "PROGRESS",
- parsed: totalParsed,
- skipped: totalSkipped,
- errors: totalErrors,
- };
- self.postMessage(message);
-}
-
function handleParsedRow(row: unknown, rowIndex: number) {
const rawEvent = row as Record<string, unknown>;
@@ -160,15 +141,9 @@ function handleParsedRow(row: unknown, rowIndex: number) {
totalParsed++;
// Send batch when it reaches chunk size
- if (currentBatch.length >= CHUNK_SIZE) {
+ if (currentBatch.length >= BATCH_SIZE) {
sendChunk();
}
-
- // Send progress update periodically
- if (totalParsed - lastProgressUpdate >= PROGRESS_UPDATE_INTERVAL) {
- sendProgress();
- lastProgressUpdate = totalParsed;
- }
}
function parseCSV(file: File) {
@@ -189,29 +164,18 @@ function parseCSV(file: File) {
}
if (results.errors && results.errors.length > 0) {
totalErrors++;
- if (errorDetails.length < 100) {
- // Limit error collection to avoid memory issues
- errorDetails.push({
- row: rowIndex,
- message: results.errors.map(e => e.message).join(", "),
- });
- }
}
},
complete: () => {
// Send final chunk if any
sendChunk();
- // Send final progress
- sendProgress();
-
// Send completion message
const message: WorkerMessageToMain = {
type: "COMPLETE",
- totalParsed,
- totalSkipped,
- totalErrors,
- errorDetails,
+ parsed: totalParsed,
+ skipped: totalSkipped,
+ errors: totalErrors,
};
self.postMessage(message);
},
@@ -237,8 +201,6 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {
totalParsed = 0;
totalSkipped = 0;
totalErrors = 0;
- errorDetails = [];
- lastProgressUpdate = 0;
// Set up quota-based date range filter
createDateRangeFilter(message.earliestAllowedDate, message.latestAllowedDate);