🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify
Edward Cui5582778aa7f3
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex f6637869e..159987ef8 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -27,7 +27,6 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/aimport { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";import { IS_CLOUD } from "@/lib/const";import { CSVWorkerManager } from "@/lib/import/csv-worker-manager";-import type { ImportProgress } from "@/lib/import/types";interface ImportManagerProps {siteId: number;@@ -80,8 +79,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const [pendingImportData, setPendingImportData] = useState<ImportFormData | null>(null);const [deleteImportId, setDeleteImportId] = useState<string | null>(null);const [showDeleteDialog, setShowDeleteDialog] = useState(false);- const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);- const [showProgressDialog, setShowProgressDialog] = useState(false);const workerManagerRef = useRef<CSVWorkerManager | null>(null);const { data, isLoading, error } = useGetSiteImports(siteId);@@ -134,20 +131,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const { importId, allowedDateRange } = response.data;// Step 2: Initialize worker manager- workerManagerRef.current = new CSVWorkerManager(- progress => {- setImportProgress(progress);- },- (success, message) => {- console.log(success ? "✅" : "❌", message);- // Import list will auto-refresh due to polling in useGetSiteImports- }- );-- // Step 3: Show progress dialog- setShowProgressDialog(true);+ workerManagerRef.current = new CSVWorkerManager();- // Step 4: Start CSV parsing and upload with allowed date range+ // Step 3: Start CSV parsing and upload with allowed date rangeworkerManagerRef.current.startImport(file,siteId,@@ -492,61 +478,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</AlertDialogFooter></AlertDialogContent></AlertDialog>-- {/* Import Progress Dialog */}- <AlertDialog open={showProgressDialog} onOpenChange={setShowProgressDialog}>- <AlertDialogContent className="max-w-2xl">- <AlertDialogHeader>- <AlertDialogTitle>Import Progress</AlertDialogTitle>- <AlertDialogDescription>- {importProgress?.status === "parsing" && "Parsing CSV file..."}- {importProgress?.status === "uploading" && "Uploading batches to server..."}- {importProgress?.status === "completed" && "Import completed!"}- {importProgress?.status === "failed" && "Import failed"}- </AlertDialogDescription>- </AlertDialogHeader>-- <div className="space-y-4">- {/* Progress Stats */}- <div className="grid grid-cols-3 gap-4">- <div>- <p className="text-sm text-muted-foreground">Parsed Rows</p>- <p className="text-2xl font-bold">{importProgress?.parsedRows.toLocaleString()}</p>- </div>- <div>- <p className="text-sm text-muted-foreground">Skipped Rows</p>- <p className="text-2xl font-bold">{importProgress?.skippedRows.toLocaleString()}</p>- </div>- <div>- <p className="text-sm text-muted-foreground">Imported Events</p>- <p className="text-2xl font-bold">{importProgress?.importedEvents.toLocaleString()}</p>- </div>- </div>-- {/* Errors */}- {importProgress && importProgress.errors > 0 && (- <div>- <p className="text-sm font-medium text-red-600">{importProgress.errors} parse errors occurred</p>- </div>- )}- </div>-- <AlertDialogFooter>- {importProgress?.status === "completed" || importProgress?.status === "failed" ? (- <AlertDialogAction onClick={() => setShowProgressDialog(false)}>Close</AlertDialogAction>- ) : (- <AlertDialogCancel- onClick={() => {- workerManagerRef.current?.terminate();- setShowProgressDialog(false);- }}- >- Cancel Import- </AlertDialogCancel>- )}- </AlertDialogFooter>- </AlertDialogContent>- </AlertDialog></div>);}diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 211f13ef9..4bb700557 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -1,12 +1,7 @@-// CSV import manager using PapaParse with built-in worker-// Handles streaming, batching, and sequential uploads-// Server handles transformation and quota checking-import Papa from "papaparse";import { DateTime } from "luxon";-import type { ImportProgress, UmamiEvent } from "./types";+import type { UmamiEvent } from "./types";-type ProgressCallback = (progress: ImportProgress) => void;type CompleteCallback = (success: boolean, message: string) => void;const BATCH_SIZE = 5000;@@ -52,17 +47,10 @@ const umamiHeaders = [export class CSVWorkerManager {private aborted = false;- private progress: ImportProgress = {- status: "idle",- parsedRows: 0,- skippedRows: 0,- importedEvents: 0,- errors: 0,- };- private onProgress: ProgressCallback | null = null;private onComplete: CompleteCallback | null = null;private uploadInProgress = false;private parsingComplete = false;+ private quotaExceeded = false;private siteId: number = 0;private importId: string = "";@@ -70,8 +58,7 @@ export class CSVWorkerManager {private earliestAllowedDate: DateTime | null = null;private latestAllowedDate: DateTime | null = null;- constructor(onProgress?: ProgressCallback, onComplete?: CompleteCallback) {- this.onProgress = onProgress || null;+ constructor(onComplete?: CompleteCallback) {this.onComplete = onComplete || null;}@@ -88,14 +75,7 @@ export class CSVWorkerManager {this.uploadInProgress = false;this.currentBatch = [];this.aborted = false;-- this.progress = {- status: "parsing",- parsedRows: 0,- skippedRows: 0,- importedEvents: 0,- errors: 0,- };+ this.quotaExceeded = false;// Set up date range filterthis.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");@@ -121,14 +101,11 @@ export class CSVWorkerManager {return umamiHeaders[index] || header;},step: results => {- if (this.aborted) return;+ if (this.aborted || this.quotaExceeded) return;if (results.data) {this.handleParsedRow(results.data);}- if (results.errors && results.errors.length > 0) {- this.progress.errors++;- }},complete: () => {if (this.aborted) return;@@ -139,8 +116,6 @@ export class CSVWorkerManager {this.handleError(error.message);},});-- this.notifyProgress();}private isDateInRange(dateStr: string): boolean {@@ -188,58 +163,63 @@ export class CSVWorkerManager {// Skip rows with missing created_at (required field)if (!umamiEvent.created_at) {- this.progress.skippedRows++;return;}// Apply quota-based date range filterif (!this.isDateInRange(umamiEvent.created_at)) {- this.progress.skippedRows++;return;}// Add to batchthis.currentBatch.push(umamiEvent);- this.progress.parsedRows++;// Send batch when it reaches chunk sizeif (this.currentBatch.length >= BATCH_SIZE) {- this.sendBatch();+ this.sendBatch(false);}}- private sendBatch(): void {+ private sendBatch(isLastBatch: boolean): void {if (this.currentBatch.length > 0) {const batch = this.currentBatch;this.currentBatch = [];-- // Upload batch (sequential)- this.progress.status = "uploading";- this.notifyProgress();- this.uploadBatch(batch);+ this.uploadBatch(batch, isLastBatch);+ } else if (isLastBatch) {+ // No events in final batch, send empty batch with finalization+ this.uploadBatch([], isLastBatch);}}private handleParseComplete(): void {this.parsingComplete = true;- // Send final batch if any- this.sendBatch();+ // Send final batch with finalization flag+ this.sendBatch(true);- // Check if upload is complete- this.checkCompletion();+ // If nothing to upload, check completion immediately+ if (!this.uploadInProgress) {+ this.checkCompletion();+ }}private handleError(message: string): void {- this.progress.status = "failed";- this.progress.errors++;- this.notifyProgress();if (this.onComplete) {this.onComplete(false, message);}}- private async uploadBatch(events: UmamiEvent[]): Promise<void> {+ private async uploadBatch(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {+ // Don't upload if quota exceeded+ if (this.quotaExceeded) {+ return;+ }++ // Skip empty batches unless it's the last one (needed for finalization)+ if (events.length === 0 && !isLastBatch) {+ return;+ }+this.uploadInProgress = true;try {@@ -251,6 +231,7 @@ export class CSVWorkerManager {credentials: "include",body: JSON.stringify({events,+ isLastBatch,}),});@@ -260,18 +241,21 @@ export class CSVWorkerManager {}const data = await response.json();- this.progress.importedEvents += data.importedCount ?? events.length;- this.notifyProgress();- } catch (error) {- // Fail fast - no retries- this.progress.status = "failed";- this.progress.errors++;- this.notifyProgress();+ // Check if quota was exceeded+ if (data.quotaExceeded) {+ this.quotaExceeded = true;+ this.aborted = true; // Stop parsing+ if (this.onComplete) {+ this.onComplete(true, data.message || "Import completed with quota limits");+ }+ return;+ }+ } catch (error) {+ // Critical failure - network error, server error, etc.if (this.onComplete) {this.onComplete(false, `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`);}-this.terminate();return;} finally {@@ -283,25 +267,13 @@ export class CSVWorkerManager {}private checkCompletion(): void {- if (this.parsingComplete && !this.uploadInProgress) {- this.progress.status = "completed";- this.notifyProgress();+ if (this.parsingComplete && !this.uploadInProgress && !this.quotaExceeded) {if (this.onComplete) {- this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);+ this.onComplete(true, "Import completed successfully");}}}- private notifyProgress(): void {- if (this.onProgress) {- this.onProgress({ ...this.progress });- }- }-- getProgress(): ImportProgress {- return { ...this.progress };- }-terminate(): void {this.aborted = true;}diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex 94c98389a..93df3f5bf 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -29,22 +29,14 @@ export interface UmamiEvent {// Batch import request (client sends raw rows to server)export interface BatchImportRequest {events: UmamiEvent[]; // Raw Umami CSV rows- importId: string;+ isLastBatch?: boolean; // Signal that this is the final batch}// Batch import responseexport interface BatchImportResponse {success: boolean;importedCount: number;+ quotaExceeded?: boolean; // Indicates quota was exceeded for this batchmessage?: string;error?: string;}--// Import progress tracking-export interface ImportProgress {- status: "idle" | "parsing" | "uploading" | "completed" | "failed";- parsedRows: number;- skippedRows: number;- importedEvents: number;- errors: number;-}diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 20b7faf8d..c44d65a66 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -17,6 +17,7 @@ const batchImportRequestSchema = z}),body: z.object({events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),+ isLastBatch: z.boolean().optional(),}),}).strict();@@ -38,7 +39,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque}const { site, importId } = parsed.data.params;- const { events } = parsed.data.body;+ const { events, isLastBatch } = parsed.data.body;const siteId = Number(site);const userHasAccess = await getUserHasAdminAccessToSite(request, site);@@ -112,20 +113,29 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeif (eventsWithinQuota.length === 0) {const quotaSummary = quotaTracker.getSummary();- const errorMessage =+ const quotaMessage =`All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;- return reply.status(400).send({- success: false,- error: "Quota exceeded",- message: errorMessage,+ if (isLastBatch) {+ await updateImportStatus(importId, "completed", quotaMessage);+ }++ return reply.send({+ success: true,+ importedCount: 0,+ quotaExceeded: true,+ message: quotaMessage,});}const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);if (transformedEvents.length === 0) {+ if (isLastBatch) {+ await updateImportStatus(importId, "completed", "No valid events found in the final batch");+ }+return reply.send({success: true,importedCount: 0,@@ -141,16 +151,27 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeawait updateImportProgress(importId, transformedEvents.length);+ if (isLastBatch) {+ const finalMessage =+ skippedDueToQuota > 0+ ? `Import completed. ${skippedDueToQuota} events were skipped due to quota limits.`+ : undefined;+ await updateImportStatus(importId, "completed", finalMessage);+ }+return reply.send({success: true,importedCount: transformedEvents.length,message: `Imported ${transformedEvents.length} events${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} skipped due to quota)` : ""}`,});} catch (insertError) {+ const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";+ await updateImportStatus(importId, "failed", `Failed to insert events: ${errorMessage}`);+return reply.status(500).send({success: false,error: "Failed to insert events",- message: insertError instanceof Error ? insertError.message : "Unknown error",+ message: errorMessage,});}} catch (error) {diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.tsindex 94c2730b7..beada2d09 100644--- a/server/src/services/import/importQuotaChecker.ts+++ b/server/src/services/import/importQuotaChecker.ts@@ -6,9 +6,6 @@ import { eq } from "drizzle-orm";import { processResults } from "../../api/analytics/utils.js";import { getBestSubscription, type SubscriptionInfo } from "../../lib/subscriptionUtils.js";import { IS_CLOUD } from "../../lib/const.js";-import { createServiceLogger } from "../../lib/logger/logger.js";--const logger = createServiceLogger("import:quota-checker");/*** Get the number of months of historical data allowed for import based on subscription tier@@ -159,7 +156,6 @@ export class ImportQuotaTracker {return monthlyUsage;} catch (error) {- logger.error({ error }, "Error querying ClickHouse for monthly usage");throw new Error(`Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`);@@ -173,7 +169,11 @@ export class ImportQuotaTracker {const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });if (!dt.isValid) {- logger.warn({ timestamp }, "Invalid timestamp format");+ return false;+ }++ const now = DateTime.utc();+ if (dt > now) {return false;}diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.tsindex 95ff282a7..8464335d0 100644--- a/server/src/services/import/mappings/umami.ts+++ b/server/src/services/import/mappings/umami.ts@@ -2,11 +2,8 @@ import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";import { getChannel } from "../../tracker/getChannel.js";import { RybbitEvent } from "./rybbit.js";import { z } from "zod";-import { createServiceLogger } from "../../../lib/logger/logger.js";import { deriveKeyOnlySchema } from "./utils.js";-const logger = createServiceLogger("import:umami-mapper");-export interface UmamiEvent {// website_id: string; // Ignoresession_id: string;@@ -161,7 +158,6 @@ export class UmamiImportMapper {return events.reduce<RybbitEvent[]>((acc, event) => {const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);if (!parsed.success) {- logger.error({ site, importId, error: parsed.error.flatten() }, "Invalid Umami event");return acc;}