🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify CSV parsing
Edward Cui490346df7e41
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex a05213b60..c1d3587b3 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -1,14 +1,13 @@"use client";import { useMemo, useState, useRef, useEffect } from "react";-import { useForm, Controller } from "react-hook-form";+import { useForm } from "react-hook-form";import { zodResolver } from "@hookform/resolvers/zod";import { z } from "zod";import { DateTime } from "luxon";import { Button } from "@/components/ui/button";import { Input } from "@/components/ui/input";import { Label } from "@/components/ui/label";-import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";import {AlertDialog,@@ -23,10 +22,8 @@ import {import { Alert, AlertDescription } from "@/components/ui/alert";import { Badge } from "@/components/ui/badge";import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";-import { Separator } from "@/components/ui/separator";import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Trash2, Info } from "lucide-react";import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/api/admin/import";-import { SplitDateRangePicker } from "@/components/SplitDateRangePicker";import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";import { IS_CLOUD } from "@/lib/const";import { CSVWorkerManager } from "@/lib/import/csv-worker-manager";@@ -63,34 +60,6 @@ const importFormSchema = z.object({const file = files[0];return file && file.name.length <= 255;}, "Filename is too long"),- dateRange: z- .object({- startDate: z.custom<DateTime>().optional(),- endDate: z.custom<DateTime>().optional(),- })- .refine(- data => {- if (!data.startDate || !data.endDate) return true;- return data.startDate <= data.endDate;- },- { message: "Start date must be before or equal to end date" }- )- .refine(- data => {- if (!data.startDate) return true;- const today = DateTime.utc().startOf("day");- return data.startDate <= today;- },- { message: "Start date cannot be in the future" }- )- .refine(- data => {- if (!data.endDate) return true;- const today = DateTime.utc().startOf("day");- return data.endDate <= today;- },- { message: "End date cannot be in the future" }- ),});type ImportFormData = z.infer<typeof importFormSchema>;@@ -120,7 +89,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const deleteMutation = useDeleteSiteImport(siteId);const {- control,register,handleSubmit,watch,@@ -131,7 +99,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {mode: "onChange",defaultValues: {file: Object.assign(new DataTransfer().files, {}),- dateRange: {},},});@@ -161,9 +128,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const file = data.file[0];if (!file) return;- const startDate = data.dateRange.startDate?.toFormat("yyyy-MM-dd");- const endDate = data.dateRange.endDate?.toFormat("yyyy-MM-dd");-// Step 1: Create import record and get allowed date rangecreateImportMutation.mutate(undefined, {onSuccess: response => {@@ -189,9 +153,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {siteId,importId,allowedDateRange.earliestAllowedDate,- allowedDateRange.latestAllowedDate,- startDate,- endDate+ allowedDateRange.latestAllowedDate);// Reset form@@ -303,28 +265,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {)}<form onSubmit={handleSubmit(onSubmit)} className="space-y-4">- {/* Date Range Picker */}- <Controller- name="dateRange"- control={control}- render={({ field }) => (- <div className="space-y-2">- <SplitDateRangePicker- value={field.value}- onChange={field.onChange}- label="Date Range (Optional)"- disabled={disabled || createImportMutation.isPending || hasActiveImport}- showDescription={true}- clearButtonText="Clear dates"- className="space-y-2"- />- {errors.dateRange?.message && <p className="text-sm text-red-600">{errors.dateRange.message}</p>}- </div>- )}- />-- <Separator />-{/* File Upload */}<div className="space-y-2"><Label htmlFor="file" className="flex items-center gap-2">@@ -568,7 +508,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<div className="space-y-4">{/* Progress Stats */}- <div className="grid grid-cols-2 gap-4">+ <div className="grid grid-cols-3 gap-4"><div><p className="text-sm text-muted-foreground">Parsed Rows</p><p className="text-2xl font-bold">{importProgress?.parsedRows.toLocaleString()}</p>@@ -577,37 +517,18 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<p className="text-sm text-muted-foreground">Skipped Rows</p><p className="text-2xl font-bold">{importProgress?.skippedRows.toLocaleString()}</p></div>- <div>- <p className="text-sm text-muted-foreground">Uploaded Batches</p>- <p className="text-2xl font-bold">- {importProgress?.uploadedBatches} / {importProgress?.totalBatches}- </p>- </div><div><p className="text-sm text-muted-foreground">Imported Events</p><p className="text-2xl font-bold">{importProgress?.importedEvents.toLocaleString()}</p></div></div>- {/* Progress Bar */}- {importProgress && importProgress.totalBatches > 0 && (- <div className="w-full bg-gray-200 rounded-full h-2.5">- <div- className="bg-blue-600 h-2.5 rounded-full transition-all"- style={{- width: `${(importProgress.uploadedBatches / importProgress.totalBatches) * 100}%`,- }}- />- </div>- )}-{/* Errors */}{importProgress && importProgress.errors.length > 0 && (<div className="max-h-40 overflow-y-auto"><p className="text-sm font-medium text-red-600 mb-2">Errors ({importProgress.errors.length}):</p>{importProgress.errors.slice(0, 10).map((error, idx) => (<p key={idx} className="text-xs text-red-600">- {error.batch !== undefined ? `Batch ${error.batch}: ` : ""}{error.message}</p>))}diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 4b0ea066a..5761e22d8 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -1,12 +1,8 @@// Worker manager for coordinating CSV import process-// Manages worker lifecycle, batch uploads, and error handling+// Manages worker lifecycle and sequential batch uploads// Server handles transformation and quota checking-import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, FailedBatch, UmamiEvent } from "./types";--const MAX_CONCURRENT_UPLOADS = 3; // Maximum number of parallel batch uploads-const RETRY_ATTEMPTS = 3; // Number of retry attempts for failed batches-const RETRY_DELAY_MS = 1000; // Initial delay between retries (exponential backoff)+import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, UmamiEvent } from "./types";type ProgressCallback = (progress: ImportProgress) => void;type CompleteCallback = (success: boolean, message: string) => void;@@ -18,18 +14,12 @@ export class CSVWorkerManager {totalRows: 0,parsedRows: 0,skippedRows: 0,- uploadedBatches: 0,- totalBatches: 0,- failedBatches: 0,importedEvents: 0,errors: [],};private onProgress: ProgressCallback | null = null;private onComplete: CompleteCallback | null = null;- private uploadQueue: Array<{ events: UmamiEvent[]; chunkIndex: number }> = [];- private activeUploads = 0;- private completedUploads = new Set<number>();- private failedBatches: FailedBatch[] = [];+ private uploadInProgress = false;private parsingComplete = false;private siteId: number = 0;private importId: string = "";@@ -44,26 +34,18 @@ export class CSVWorkerManager {siteId: number,importId: string,earliestAllowedDate: string,- latestAllowedDate: string,- startDate?: string,- endDate?: string+ latestAllowedDate: string): void {this.siteId = siteId;this.importId = importId;this.parsingComplete = false;- this.uploadQueue = [];- this.activeUploads = 0;- this.completedUploads.clear();- this.failedBatches = [];+ this.uploadInProgress = false;this.progress = {status: "parsing",totalRows: 0,parsedRows: 0,skippedRows: 0,- uploadedBatches: 0,- totalBatches: 0,- failedBatches: 0,importedEvents: 0,errors: [],};@@ -90,7 +72,7 @@ export class CSVWorkerManager {}};- // Start parsing with allowed date range for client-side filtering+ // Start parsing with quota-based date range for client-side filteringconst message: WorkerMessageToWorker = {type: "PARSE_START",file,@@ -98,8 +80,6 @@ export class CSVWorkerManager {importId,earliestAllowedDate,latestAllowedDate,- startDate,- endDate,};this.worker.postMessage(message);@@ -119,17 +99,10 @@ export class CSVWorkerManager {break;case "CHUNK_READY":- // Add to upload queue- this.uploadQueue.push({- events: message.events,- chunkIndex: message.chunkIndex,- });- this.progress.totalBatches++;+ // Upload batch immediately (sequential)this.progress.status = "uploading";this.notifyProgress();-- // Process upload queue- this.processUploadQueue();+ this.uploadBatch(message.events);break;case "COMPLETE":@@ -146,7 +119,7 @@ export class CSVWorkerManager {this.notifyProgress();- // Check if all uploads are complete+ // Check if upload is completethis.checkCompletion();break;@@ -166,18 +139,9 @@ export class CSVWorkerManager {}}- private async processUploadQueue(): Promise<void> {- // Process queued uploads with concurrency limit- while (this.uploadQueue.length > 0 && this.activeUploads < MAX_CONCURRENT_UPLOADS) {- const batch = this.uploadQueue.shift();- if (!batch) continue;-- this.activeUploads++;- this.uploadBatch(batch.events, batch.chunkIndex);- }- }+ private async uploadBatch(events: UmamiEvent[]): Promise<void> {+ this.uploadInProgress = true;- private async uploadBatch(events: UmamiEvent[], chunkIndex: number, retryCount = 0): Promise<void> {try {const response = await fetch(`/api/batch-import-events/${this.siteId}`, {method: "POST",@@ -188,8 +152,6 @@ export class CSVWorkerManager {body: JSON.stringify({events,importId: this.importId,- batchIndex: chunkIndex,- totalBatches: this.progress.totalBatches,}),});@@ -199,77 +161,36 @@ export class CSVWorkerManager {}const data = await response.json();-- // Mark as completed- this.completedUploads.add(chunkIndex);- this.progress.uploadedBatches++;this.progress.importedEvents += data.importedCount ?? events.length;-- this.activeUploads--;this.notifyProgress();-- // Continue processing queue- this.processUploadQueue();-- // Check if we're done- this.checkCompletion();} catch (error) {- console.error(`Failed to upload batch ${chunkIndex}:`, error);-- // Retry with exponential backoff- if (retryCount < RETRY_ATTEMPTS) {- const delay = RETRY_DELAY_MS * Math.pow(2, retryCount);- console.log(`Retrying batch ${chunkIndex} in ${delay}ms (attempt ${retryCount + 1}/${RETRY_ATTEMPTS})`);-- setTimeout(() => {- this.uploadBatch(events, chunkIndex, retryCount + 1);- }, delay);- } else {- // Max retries exceeded, record as failed- this.failedBatches.push({- batchIndex: chunkIndex,- events,- error: error instanceof Error ? error.message : "Unknown error",- retryCount,- });-- this.progress.failedBatches++;- this.progress.errors.push({- batch: chunkIndex,- message: `Batch ${chunkIndex} failed after ${RETRY_ATTEMPTS} attempts: ${error instanceof Error ? error.message : "Unknown error"}`,- });-- this.activeUploads--;- this.notifyProgress();+ // Fail fast - no retries+ this.progress.status = "failed";+ this.progress.errors.push({+ message: `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`,+ });+ this.notifyProgress();- // Continue processing other batches- this.processUploadQueue();- this.checkCompletion();+ if (this.onComplete) {+ this.onComplete(false, `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`);}++ this.terminate();+ return;+ } finally {+ this.uploadInProgress = false;}++ // Check if we're done+ this.checkCompletion();}private checkCompletion(): void {- if (this.parsingComplete && this.activeUploads === 0 && this.uploadQueue.length === 0) {- const totalExpectedBatches = this.progress.totalBatches;- const successfulBatches = this.progress.uploadedBatches;- const failedCount = this.progress.failedBatches;-- if (failedCount === 0) {- this.progress.status = "completed";- this.notifyProgress();- if (this.onComplete) {- this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);- }- } else {- this.progress.status = "completed";- this.notifyProgress();- if (this.onComplete) {- this.onComplete(- false,- `Import completed with errors: ${successfulBatches}/${totalExpectedBatches} batches succeeded, ${failedCount} batches failed`- );- }+ if (this.parsingComplete && !this.uploadInProgress) {+ this.progress.status = "completed";+ this.notifyProgress();+ if (this.onComplete) {+ this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);}// Clean up worker@@ -287,10 +208,6 @@ export class CSVWorkerManager {return { ...this.progress };}- getFailedBatches(): FailedBatch[] {- return [...this.failedBatches];- }-terminate(): void {if (this.worker) {const message: WorkerMessageToWorker = {diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex 0ee73a8ed..bceec538b 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -38,10 +38,8 @@ export type WorkerMessageToWorker =file: File;siteId: number;importId: string;- earliestAllowedDate: string; // Filter events before this date (yyyy-MM-dd)- latestAllowedDate: string; // Filter events after this date (yyyy-MM-dd)- startDate?: string; // Optional user-specified start date filter- endDate?: string; // Optional user-specified end date filter+ earliestAllowedDate: string; // Filter events before this date (quota-based, yyyy-MM-dd)+ latestAllowedDate: string; // Filter events after this date (quota-based, yyyy-MM-dd)}| {type: "CANCEL";@@ -57,7 +55,6 @@ export type WorkerMessageToMain =| {type: "CHUNK_READY";events: UmamiEvent[]; // Raw CSV rows, not transformed- chunkIndex: number;}| {type: "COMPLETE";@@ -76,8 +73,6 @@ export type WorkerMessageToMain =export interface BatchImportRequest {events: UmamiEvent[]; // Raw Umami CSV rowsimportId: string;- batchIndex: number;- totalBatches: number;}// Batch import response@@ -94,17 +89,6 @@ export interface ImportProgress {totalRows: number;parsedRows: number;skippedRows: number;- uploadedBatches: number;- totalBatches: number;- failedBatches: number;importedEvents: number;- errors: Array<{ batch?: number; message: string }>;-}--// Failed batch info for retry-export interface FailedBatch {- batchIndex: number;- events: UmamiEvent[]; // Raw rows, not transformed- error: string;- retryCount: number;+ errors: Array<{ message: string }>;}diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsindex 244e8637a..4b2ffcb6c 100644--- a/client/src/workers/csv-import.worker.ts+++ b/client/src/workers/csv-import.worker.ts@@ -10,7 +10,6 @@ const CHUNK_SIZE = 5000; // Number of rows per batch sent to main threadconst PROGRESS_UPDATE_INTERVAL = 1000; // Update progress every 1000 rowslet currentBatch: UmamiEvent[] = [];-let chunkIndex = 0;let totalParsed = 0;let totalSkipped = 0;let totalErrors = 0;@@ -18,12 +17,9 @@ let errorDetails: Array<{ row: number; message: string }> = [];let lastProgressUpdate = 0;// Date range filters (client-side optimization to reduce server load)-// Quota-enforced dates (required based on subscription tier)+// Quota-enforced dates (based on subscription tier)let earliestAllowedDate: DateTime | null = null;let latestAllowedDate: DateTime | null = null;-// User-specified optional date filters-let userStartDate: DateTime | null = null;-let userEndDate: DateTime | null = null;// Umami CSV header mappingconst umamiHeaders = [@@ -65,13 +61,8 @@ const umamiHeaders = [undefined,];-function createDateRangeFilter(- earliestAllowedDateStr: string,- latestAllowedDateStr: string,- userStartDateStr?: string,- userEndDateStr?: string-) {- // Parse quota-enforced dates (required)+function createDateRangeFilter(earliestAllowedDateStr: string, latestAllowedDateStr: string) {+ // Parse quota-enforced datesearliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");@@ -82,22 +73,6 @@ function createDateRangeFilter(if (!latestAllowedDate.isValid) {throw new Error(`Invalid latest allowed date: ${latestAllowedDateStr}`);}-- // Parse user-specified optional dates- userStartDate = userStartDateStr- ? DateTime.fromFormat(userStartDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")- : null;- userEndDate = userEndDateStr- ? DateTime.fromFormat(userEndDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")- : null;-- if (userStartDate && !userStartDate.isValid) {- throw new Error(`Invalid user start date: ${userStartDateStr}`);- }-- if (userEndDate && !userEndDate.isValid) {- throw new Error(`Invalid user end date: ${userEndDateStr}`);- }}function isDateInRange(dateStr: string): boolean {@@ -106,7 +81,7 @@ function isDateInRange(dateStr: string): boolean {return false;}- // Check quota-enforced dates (required)+ // Check quota-enforced datesif (earliestAllowedDate && createdAt < earliestAllowedDate) {return false;}@@ -115,15 +90,6 @@ function isDateInRange(dateStr: string): boolean {return false;}- // Check user-specified optional dates- if (userStartDate && createdAt < userStartDate) {- return false;- }-- if (userEndDate && createdAt > userEndDate) {- return false;- }-return true;}@@ -132,10 +98,8 @@ function sendChunk() {const message: WorkerMessageToMain = {type: "CHUNK_READY",events: currentBatch,- chunkIndex,};self.postMessage(message);- chunkIndex++;currentBatch = [];}}@@ -184,8 +148,8 @@ function handleParsedRow(row: unknown, rowIndex: number) {return;}- // Apply date range filter (quota-enforced + optional user filters)- // This client-side filtering reduces network traffic and server processing+ // Apply quota-based date range filter+ // This client-side filtering reduces network trafficif (!isDateInRange(umamiEvent.created_at)) {totalSkipped++;return;@@ -270,20 +234,14 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {case "PARSE_START":// Reset statecurrentBatch = [];- chunkIndex = 0;totalParsed = 0;totalSkipped = 0;totalErrors = 0;errorDetails = [];lastProgressUpdate = 0;- // Set up date range filters (quota-enforced + optional user filters)- createDateRangeFilter(- message.earliestAllowedDate,- message.latestAllowedDate,- message.startDate,- message.endDate- );+ // Set up quota-based date range filter+ createDateRangeFilter(message.earliestAllowedDate, message.latestAllowedDate);// Start parsingparseCSV(message.file);diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex ba80ed3fb..7d74164bf 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -44,8 +44,6 @@ const batchImportRequestSchema = zbody: z.object({events: z.array(umamiEventSchema).min(1).max(10000), // Properly typed Umami eventsimportId: z.string().uuid(),- batchIndex: z.number().int().min(0),- totalBatches: z.number().int().min(1),}),}).strict();@@ -68,7 +66,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}const { site } = parsed.data.params;- const { events, importId, batchIndex, totalBatches } = parsed.data.body;+ const { events, importId } = parsed.data.body;const siteId = Number(site);const userHasAccess = await getUserHasAdminAccessToSite(request, site);@@ -151,19 +149,14 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}}- // If all events were skipped due to quota, fail the batch+ // If all events were skipped due to quota, log warningif (eventsWithinQuota.length === 0 && events.length > 0) {const quotaSummary = quotaTracker.getSummary();const errorMessage =- `All ${events.length} events in batch ${batchIndex} exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` ++ `All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;- logger.warn({ importId, batchIndex, skippedDueToQuota }, errorMessage);-- // Update import status to failed if this was the only batch or early in the import- if (batchIndex === 0 || totalBatches === 1) {- await updateImportStatus(importId, "failed", errorMessage);- }+ logger.warn({ importId, skippedDueToQuota }, errorMessage);return reply.status(400).send({success: false,@@ -176,11 +169,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeconst transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);if (transformedEvents.length === 0) {- logger.warn({ importId, batchIndex }, "No valid events after transformation");+ logger.warn({ importId }, "No valid events after transformation");return reply.send({success: true,importedCount: 0,- message: `Batch ${batchIndex + 1}/${totalBatches}: No valid events`,+ message: "No valid events in batch",});}@@ -194,10 +187,8 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequelogger.info({importId,- batchIndex,eventCount: transformedEvents.length,skippedDueToQuota,- totalBatches,},"Batch inserted successfully");@@ -208,13 +199,11 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequereturn reply.send({success: true,importedCount: transformedEvents.length,- message: `Batch ${batchIndex + 1}/${totalBatches} imported successfully${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} events skipped due to quota)` : ""}`,+ message: `Imported ${transformedEvents.length} events${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} skipped due to quota)` : ""}`,});} catch (insertError) {- logger.error({ importId, batchIndex, error: insertError }, "Failed to insert batch");+ logger.error({ importId, error: insertError }, "Failed to insert batch");- // Don't mark entire import as failed for individual batch failures- // The client will retry and handle failuresreturn reply.status(500).send({success: false,error: "Failed to insert events",diff --git a/server/src/api/sites/getSiteImports.ts b/server/src/api/sites/getSiteImports.tsindex 2bb41f636..dc426b6b0 100644--- a/server/src/api/sites/getSiteImports.ts+++ b/server/src/api/sites/getSiteImports.ts@@ -36,14 +36,13 @@ export async function getSiteImports(request: FastifyRequest<GetSiteImportsRequeconst imports = await getImportsForSite(siteId);return reply.send({- data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt, fileName }) => ({+ data: imports.map(({ importId, platform, status, importedEvents, errorMessage, startedAt }) => ({importId,platform,status,importedEvents,errorMessage,startedAt,- fileName,})),});} catch (error) {