Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify

Edward Cui5582778aa7f3
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index f6637869e..159987ef8 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -27,7 +27,6 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/a
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { IS_CLOUD } from "@/lib/const";
import { CSVWorkerManager } from "@/lib/import/csv-worker-manager";
-import type { ImportProgress } from "@/lib/import/types";
interface ImportManagerProps {
siteId: number;
@@ -80,8 +79,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const [pendingImportData, setPendingImportData] = useState<ImportFormData | null>(null);
const [deleteImportId, setDeleteImportId] = useState<string | null>(null);
const [showDeleteDialog, setShowDeleteDialog] = useState(false);
- const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);
- const [showProgressDialog, setShowProgressDialog] = useState(false);
const workerManagerRef = useRef<CSVWorkerManager | null>(null);
const { data, isLoading, error } = useGetSiteImports(siteId);
@@ -134,20 +131,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const { importId, allowedDateRange } = response.data;
// Step 2: Initialize worker manager
- workerManagerRef.current = new CSVWorkerManager(
- progress => {
- setImportProgress(progress);
- },
- (success, message) => {
- console.log(success ? "✅" : "❌", message);
- // Import list will auto-refresh due to polling in useGetSiteImports
- }
- );
-
- // Step 3: Show progress dialog
- setShowProgressDialog(true);
+ workerManagerRef.current = new CSVWorkerManager();
- // Step 4: Start CSV parsing and upload with allowed date range
+ // Step 3: Start CSV parsing and upload with allowed date range
workerManagerRef.current.startImport(
file,
siteId,
@@ -492,61 +478,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
-
- {/* Import Progress Dialog */}
- <AlertDialog open={showProgressDialog} onOpenChange={setShowProgressDialog}>
- <AlertDialogContent className="max-w-2xl">
- <AlertDialogHeader>
- <AlertDialogTitle>Import Progress</AlertDialogTitle>
- <AlertDialogDescription>
- {importProgress?.status === "parsing" && "Parsing CSV file..."}
- {importProgress?.status === "uploading" && "Uploading batches to server..."}
- {importProgress?.status === "completed" && "Import completed!"}
- {importProgress?.status === "failed" && "Import failed"}
- </AlertDialogDescription>
- </AlertDialogHeader>
-
- <div className="space-y-4">
- {/* Progress Stats */}
- <div className="grid grid-cols-3 gap-4">
- <div>
- <p className="text-sm text-muted-foreground">Parsed Rows</p>
- <p className="text-2xl font-bold">{importProgress?.parsedRows.toLocaleString()}</p>
- </div>
- <div>
- <p className="text-sm text-muted-foreground">Skipped Rows</p>
- <p className="text-2xl font-bold">{importProgress?.skippedRows.toLocaleString()}</p>
- </div>
- <div>
- <p className="text-sm text-muted-foreground">Imported Events</p>
- <p className="text-2xl font-bold">{importProgress?.importedEvents.toLocaleString()}</p>
- </div>
- </div>
-
- {/* Errors */}
- {importProgress && importProgress.errors > 0 && (
- <div>
- <p className="text-sm font-medium text-red-600">{importProgress.errors} parse errors occurred</p>
- </div>
- )}
- </div>
-
- <AlertDialogFooter>
- {importProgress?.status === "completed" || importProgress?.status === "failed" ? (
- <AlertDialogAction onClick={() => setShowProgressDialog(false)}>Close</AlertDialogAction>
- ) : (
- <AlertDialogCancel
- onClick={() => {
- workerManagerRef.current?.terminate();
- setShowProgressDialog(false);
- }}
- >
- Cancel Import
- </AlertDialogCancel>
- )}
- </AlertDialogFooter>
- </AlertDialogContent>
- </AlertDialog>
</div>
);
}
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index 211f13ef9..4bb700557 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -1,12 +1,7 @@
-// CSV import manager using PapaParse with built-in worker
-// Handles streaming, batching, and sequential uploads
-// Server handles transformation and quota checking
-
import Papa from "papaparse";
import { DateTime } from "luxon";
-import type { ImportProgress, UmamiEvent } from "./types";
+import type { UmamiEvent } from "./types";
-type ProgressCallback = (progress: ImportProgress) => void;
type CompleteCallback = (success: boolean, message: string) => void;
const BATCH_SIZE = 5000;
@@ -52,17 +47,10 @@ const umamiHeaders = [
export class CSVWorkerManager {
private aborted = false;
- private progress: ImportProgress = {
- status: "idle",
- parsedRows: 0,
- skippedRows: 0,
- importedEvents: 0,
- errors: 0,
- };
- private onProgress: ProgressCallback | null = null;
private onComplete: CompleteCallback | null = null;
private uploadInProgress = false;
private parsingComplete = false;
+ private quotaExceeded = false;
private siteId: number = 0;
private importId: string = "";
@@ -70,8 +58,7 @@ export class CSVWorkerManager {
private earliestAllowedDate: DateTime | null = null;
private latestAllowedDate: DateTime | null = null;
- constructor(onProgress?: ProgressCallback, onComplete?: CompleteCallback) {
- this.onProgress = onProgress || null;
+ constructor(onComplete?: CompleteCallback) {
this.onComplete = onComplete || null;
}
@@ -88,14 +75,7 @@ export class CSVWorkerManager {
this.uploadInProgress = false;
this.currentBatch = [];
this.aborted = false;
-
- this.progress = {
- status: "parsing",
- parsedRows: 0,
- skippedRows: 0,
- importedEvents: 0,
- errors: 0,
- };
+ this.quotaExceeded = false;
// Set up date range filter
this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
@@ -121,14 +101,11 @@ export class CSVWorkerManager {
return umamiHeaders[index] || header;
},
step: results => {
- if (this.aborted) return;
+ if (this.aborted || this.quotaExceeded) return;
if (results.data) {
this.handleParsedRow(results.data);
}
- if (results.errors && results.errors.length > 0) {
- this.progress.errors++;
- }
},
complete: () => {
if (this.aborted) return;
@@ -139,8 +116,6 @@ export class CSVWorkerManager {
this.handleError(error.message);
},
});
-
- this.notifyProgress();
}
private isDateInRange(dateStr: string): boolean {
@@ -188,58 +163,63 @@ export class CSVWorkerManager {
// Skip rows with missing created_at (required field)
if (!umamiEvent.created_at) {
- this.progress.skippedRows++;
return;
}
// Apply quota-based date range filter
if (!this.isDateInRange(umamiEvent.created_at)) {
- this.progress.skippedRows++;
return;
}
// Add to batch
this.currentBatch.push(umamiEvent);
- this.progress.parsedRows++;
// Send batch when it reaches chunk size
if (this.currentBatch.length >= BATCH_SIZE) {
- this.sendBatch();
+ this.sendBatch(false);
}
}
- private sendBatch(): void {
+ private sendBatch(isLastBatch: boolean): void {
if (this.currentBatch.length > 0) {
const batch = this.currentBatch;
this.currentBatch = [];
-
- // Upload batch (sequential)
- this.progress.status = "uploading";
- this.notifyProgress();
- this.uploadBatch(batch);
+ this.uploadBatch(batch, isLastBatch);
+ } else if (isLastBatch) {
+ // No events in final batch, send empty batch with finalization
+ this.uploadBatch([], isLastBatch);
}
}
private handleParseComplete(): void {
this.parsingComplete = true;
- // Send final batch if any
- this.sendBatch();
+ // Send final batch with finalization flag
+ this.sendBatch(true);
- // Check if upload is complete
- this.checkCompletion();
+ // If nothing to upload, check completion immediately
+ if (!this.uploadInProgress) {
+ this.checkCompletion();
+ }
}
private handleError(message: string): void {
- this.progress.status = "failed";
- this.progress.errors++;
- this.notifyProgress();
if (this.onComplete) {
this.onComplete(false, message);
}
}
- private async uploadBatch(events: UmamiEvent[]): Promise<void> {
+ private async uploadBatch(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {
+ // Don't upload if quota exceeded
+ if (this.quotaExceeded) {
+ return;
+ }
+
+ // Skip empty batches unless it's the last one (needed for finalization)
+ if (events.length === 0 && !isLastBatch) {
+ return;
+ }
+
this.uploadInProgress = true;
try {
@@ -251,6 +231,7 @@ export class CSVWorkerManager {
credentials: "include",
body: JSON.stringify({
events,
+ isLastBatch,
}),
});
@@ -260,18 +241,21 @@ export class CSVWorkerManager {
}
const data = await response.json();
- this.progress.importedEvents += data.importedCount ?? events.length;
- this.notifyProgress();
- } catch (error) {
- // Fail fast - no retries
- this.progress.status = "failed";
- this.progress.errors++;
- this.notifyProgress();
+ // Check if quota was exceeded
+ if (data.quotaExceeded) {
+ this.quotaExceeded = true;
+ this.aborted = true; // Stop parsing
+ if (this.onComplete) {
+ this.onComplete(true, data.message || "Import completed with quota limits");
+ }
+ return;
+ }
+ } catch (error) {
+ // Critical failure - network error, server error, etc.
if (this.onComplete) {
this.onComplete(false, `Upload failed: ${error instanceof Error ? error.message : "Unknown error"}`);
}
-
this.terminate();
return;
} finally {
@@ -283,25 +267,13 @@ export class CSVWorkerManager {
}
private checkCompletion(): void {
- if (this.parsingComplete && !this.uploadInProgress) {
- this.progress.status = "completed";
- this.notifyProgress();
+ if (this.parsingComplete && !this.uploadInProgress && !this.quotaExceeded) {
if (this.onComplete) {
- this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);
+ this.onComplete(true, "Import completed successfully");
}
}
}
- private notifyProgress(): void {
- if (this.onProgress) {
- this.onProgress({ ...this.progress });
- }
- }
-
- getProgress(): ImportProgress {
- return { ...this.progress };
- }
-
terminate(): void {
this.aborted = true;
}
diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.ts
index 94c98389a..93df3f5bf 100644
--- a/client/src/lib/import/types.ts
+++ b/client/src/lib/import/types.ts
@@ -29,22 +29,14 @@ export interface UmamiEvent {
// Batch import request (client sends raw rows to server)
export interface BatchImportRequest {
events: UmamiEvent[]; // Raw Umami CSV rows
- importId: string;
+ isLastBatch?: boolean; // Signal that this is the final batch
}
// Batch import response
export interface BatchImportResponse {
success: boolean;
importedCount: number;
+ quotaExceeded?: boolean; // Indicates quota was exceeded for this batch
message?: string;
error?: string;
}
-
-// Import progress tracking
-export interface ImportProgress {
- status: "idle" | "parsing" | "uploading" | "completed" | "failed";
- parsedRows: number;
- skippedRows: number;
- importedEvents: number;
- errors: number;
-}
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 20b7faf8d..c44d65a66 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -17,6 +17,7 @@ const batchImportRequestSchema = z
}),
body: z.object({
events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema).min(1).max(10000),
+ isLastBatch: z.boolean().optional(),
}),
})
.strict();
@@ -38,7 +39,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
}
const { site, importId } = parsed.data.params;
- const { events } = parsed.data.body;
+ const { events, isLastBatch } = parsed.data.body;
const siteId = Number(site);
const userHasAccess = await getUserHasAdminAccessToSite(request, site);
@@ -112,20 +113,29 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
if (eventsWithinQuota.length === 0) {
const quotaSummary = quotaTracker.getSummary();
- const errorMessage =
+ const quotaMessage =
`All ${events.length} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
`${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;
- return reply.status(400).send({
- success: false,
- error: "Quota exceeded",
- message: errorMessage,
+ if (isLastBatch) {
+ await updateImportStatus(importId, "completed", quotaMessage);
+ }
+
+ return reply.send({
+ success: true,
+ importedCount: 0,
+ quotaExceeded: true,
+ message: quotaMessage,
});
}
const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);
if (transformedEvents.length === 0) {
+ if (isLastBatch) {
+ await updateImportStatus(importId, "completed", "No valid events found in the final batch");
+ }
+
return reply.send({
success: true,
importedCount: 0,
@@ -141,16 +151,27 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
await updateImportProgress(importId, transformedEvents.length);
+ if (isLastBatch) {
+ const finalMessage =
+ skippedDueToQuota > 0
+ ? `Import completed. ${skippedDueToQuota} events were skipped due to quota limits.`
+ : undefined;
+ await updateImportStatus(importId, "completed", finalMessage);
+ }
+
return reply.send({
success: true,
importedCount: transformedEvents.length,
message: `Imported ${transformedEvents.length} events${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} skipped due to quota)` : ""}`,
});
} catch (insertError) {
+ const errorMessage = insertError instanceof Error ? insertError.message : "Unknown error";
+ await updateImportStatus(importId, "failed", `Failed to insert events: ${errorMessage}`);
+
return reply.status(500).send({
success: false,
error: "Failed to insert events",
- message: insertError instanceof Error ? insertError.message : "Unknown error",
+ message: errorMessage,
});
}
} catch (error) {
diff --git a/server/src/services/import/importQuotaChecker.ts b/server/src/services/import/importQuotaChecker.ts
index 94c2730b7..beada2d09 100644
--- a/server/src/services/import/importQuotaChecker.ts
+++ b/server/src/services/import/importQuotaChecker.ts
@@ -6,9 +6,6 @@ import { eq } from "drizzle-orm";
import { processResults } from "../../api/analytics/utils.js";
import { getBestSubscription, type SubscriptionInfo } from "../../lib/subscriptionUtils.js";
import { IS_CLOUD } from "../../lib/const.js";
-import { createServiceLogger } from "../../lib/logger/logger.js";
-
-const logger = createServiceLogger("import:quota-checker");
/**
* Get the number of months of historical data allowed for import based on subscription tier
@@ -159,7 +156,6 @@ export class ImportQuotaTracker {
return monthlyUsage;
} catch (error) {
- logger.error({ error }, "Error querying ClickHouse for monthly usage");
throw new Error(
`Failed to query monthly usage for quota check: ${error instanceof Error ? error.message : "Unknown error"}`
);
@@ -173,7 +169,11 @@ export class ImportQuotaTracker {
const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
if (!dt.isValid) {
- logger.warn({ timestamp }, "Invalid timestamp format");
+ return false;
+ }
+
+ const now = DateTime.utc();
+ if (dt > now) {
return false;
}
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappings/umami.ts
index 95ff282a7..8464335d0 100644
--- a/server/src/services/import/mappings/umami.ts
+++ b/server/src/services/import/mappings/umami.ts
@@ -2,11 +2,8 @@ import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";
import { getChannel } from "../../tracker/getChannel.js";
import { RybbitEvent } from "./rybbit.js";
import { z } from "zod";
-import { createServiceLogger } from "../../../lib/logger/logger.js";
import { deriveKeyOnlySchema } from "./utils.js";
-const logger = createServiceLogger("import:umami-mapper");
-
export interface UmamiEvent {
// website_id: string; // Ignore
session_id: string;
@@ -161,7 +158,6 @@ export class UmamiImportMapper {
return events.reduce<RybbitEvent[]>((acc, event) => {
const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);
if (!parsed.success) {
- logger.error({ site, importId, error: parsed.error.flatten() }, "Invalid Umami event");
return acc;
}