🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Client import
Edward Cui5c5d0bab38bf
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex c06bedc56..604a789b7 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -1,6 +1,7 @@import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";import { authedFetch } from "@/api/utils";import { APIResponse } from "@/api/types";+import type { AllowedDateRange } from "@/lib/import/types";interface GetSiteImportsResponse {importId: string;@@ -12,6 +13,16 @@ interface GetSiteImportsResponse {fileName: string;}+interface CreateSiteImportParams {+ platform: "umami";+ fileName: string;+}++interface CreateSiteImportResponse {+ importId: string;+ allowedDateRange: AllowedDateRange;+}+interface DeleteImportResponse {message: string;}@@ -31,6 +42,29 @@ export function useGetSiteImports(site: number) {});}+export function useCreateSiteImport(site: number) {+ const queryClient = useQueryClient();++ return useMutation({+ mutationFn: async (params: CreateSiteImportParams) => {+ return await authedFetch<APIResponse<CreateSiteImportResponse>>(+ `/create-site-import/${site}`,+ undefined,+ {+ method: "POST",+ data: params,+ }+ );+ },+ onSuccess: async () => {+ await queryClient.invalidateQueries({+ queryKey: ["get-site-imports", site],+ });+ },+ retry: false,+ });+}+export function useDeleteSiteImport(site: number) {const queryClient = useQueryClient();diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex c2ade26a0..b7bbb5020 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -1,6 +1,6 @@"use client";-import { useMemo, useState } from "react";+import { useMemo, useState, useRef, useEffect } from "react";import { useForm, Controller } from "react-hook-form";import { zodResolver } from "@hookform/resolvers/zod";import { z } from "zod";@@ -25,10 +25,12 @@ import { Badge } from "@/components/ui/badge";import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";import { Separator } from "@/components/ui/separator";import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Database, Trash2, Info } from "lucide-react";-import { useGetSiteImports, useImportSiteData, useDeleteSiteImport } from "@/api/admin/import";+import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/api/admin/import";import { SplitDateRangePicker } from "@/components/SplitDateRangePicker";import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";import { IS_CLOUD } from "@/lib/const";+import { CSVWorkerManager } from "@/lib/import/csv-worker-manager";+import type { ImportProgress } from "@/lib/import/types";interface ImportManagerProps {siteId: number;@@ -108,11 +110,15 @@ function formatFileSize(bytes: number): string {export function ImportManager({ siteId, disabled }: ImportManagerProps) {const [showConfirmDialog, setShowConfirmDialog] = useState(false);+ const [pendingImportData, setPendingImportData] = useState<ImportFormData | null>(null);const [deleteImportId, setDeleteImportId] = useState<string | null>(null);const [showDeleteDialog, setShowDeleteDialog] = useState(false);+ const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);+ const [showProgressDialog, setShowProgressDialog] = useState(false);+ const workerManagerRef = useRef<CSVWorkerManager | null>(null);const { data, isLoading, error } = useGetSiteImports(siteId);- const mutation = useImportSiteData(siteId);+ const createImportMutation = useCreateSiteImport(siteId);const deleteMutation = useDeleteSiteImport(siteId);const {@@ -135,11 +141,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const fileList = watch("file");const selectedFile = fileList?.[0];+ // Cleanup worker on component unmount+ useEffect(() => {+ return () => {+ workerManagerRef.current?.terminate();+ };+ }, []);+const onSubmit = (data: ImportFormData) => {const file = data.file[0];if (!file) return;if (file.size > CONFIRM_THRESHOLD) {+ setPendingImportData(data);setShowConfirmDialog(true);} else {executeImport(data);@@ -153,17 +167,48 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const startDate = data.dateRange.startDate?.toFormat("yyyy-MM-dd");const endDate = data.dateRange.endDate?.toFormat("yyyy-MM-dd");- mutation.mutate(+ // Step 1: Create import record and get allowed date range+ createImportMutation.mutate({- file,platform: data.platform,- startDate,- endDate,+ fileName: file.name,},{- onSuccess: () => {+ onSuccess: response => {+ const { importId, allowedDateRange } = response.data;++ // Step 2: Initialize worker manager+ workerManagerRef.current = new CSVWorkerManager(+ progress => {+ setImportProgress(progress);+ },+ (success, message) => {+ console.log(success ? "✅" : "❌", message);+ // Import list will auto-refresh due to polling in useGetSiteImports+ }+ );++ // Step 3: Show progress dialog+ setShowProgressDialog(true);++ // Step 4: Start CSV parsing and upload with allowed date range+ workerManagerRef.current.startImport(+ file,+ siteId,+ importId,+ data.platform,+ allowedDateRange.earliestAllowedDate,+ allowedDateRange.latestAllowedDate,+ startDate,+ endDate+ );++ // Reset formreset();},+ onError: error => {+ console.error("Failed to create import:", error);+ },});@@ -237,13 +282,11 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {});}, [data?.data]);- const formData = watch();-// Check if there's an active import (cloud only)const hasActiveImport =IS_CLOUD && sortedImports.some(imp => imp.status === "pending" || imp.status === "processing");- const isImportDisabled = !isValid || mutation.isPending || disabled || hasActiveImport;+ const isImportDisabled = !isValid || createImportMutation.isPending || disabled || hasActiveImport;return (<div className="space-y-6">@@ -283,7 +326,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Selectvalue={field.value}onValueChange={field.onChange}- disabled={disabled || mutation.isPending || hasActiveImport}+ disabled={disabled || createImportMutation.isPending || hasActiveImport}><SelectTrigger><SelectValue placeholder="Select platform" />@@ -311,7 +354,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {value={field.value}onChange={field.onChange}label="Date Range (Optional)"- disabled={disabled || mutation.isPending || hasActiveImport}+ disabled={disabled || createImportMutation.isPending || hasActiveImport}showDescription={true}clearButtonText="Clear dates"className="space-y-2"@@ -334,7 +377,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {accept=".csv"multiple={false}{...register("file")}- disabled={disabled || mutation.isPending || hasActiveImport}+ disabled={disabled || createImportMutation.isPending || hasActiveImport}/>{selectedFile && (<p className="text-sm text-muted-foreground">@@ -346,7 +389,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {{/* Import Button */}<Button type="submit" disabled={isImportDisabled} className="w-full sm:w-auto">- {mutation.isPending ? (+ {createImportMutation.isPending ? (<><Loader2 className="mr-2 h-4 w-4 animate-spin" />Importing...@@ -361,19 +404,19 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</form>{/* Import Error */}- {mutation.isError && (+ {createImportMutation.isError && (<Alert variant="destructive"><div className="flex items-center gap-2"><AlertCircle className="h-4 w-4" /><AlertDescription>- {mutation.error.message || "Failed to import file. Please try again."}+ {createImportMutation.error.message || "Failed to import file. Please try again."}</AlertDescription></div></Alert>)}{/* Success Message */}- {mutation.isSuccess && (+ {createImportMutation.isSuccess && (<Alert className="border-green-200 bg-green-50"><div className="flex items-center gap-2"><CheckCircle2 className="h-4 w-4 text-green-600" />@@ -528,7 +571,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</AlertDialogHeader><AlertDialogFooter><AlertDialogCancel>Cancel</AlertDialogCancel>- <AlertDialogAction onClick={() => executeImport(formData)}>Yes, Import File</AlertDialogAction>+ <AlertDialogAction onClick={() => pendingImportData && executeImport(pendingImportData)}>+ Yes, Import File+ </AlertDialogAction></AlertDialogFooter></AlertDialogContent></AlertDialog>@@ -551,6 +596,88 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {</AlertDialogFooter></AlertDialogContent></AlertDialog>++ {/* Import Progress Dialog */}+ <AlertDialog open={showProgressDialog} onOpenChange={setShowProgressDialog}>+ <AlertDialogContent className="max-w-2xl">+ <AlertDialogHeader>+ <AlertDialogTitle>Import Progress</AlertDialogTitle>+ <AlertDialogDescription>+ {importProgress?.status === "parsing" && "Parsing CSV file..."}+ {importProgress?.status === "uploading" && "Uploading batches to server..."}+ {importProgress?.status === "completed" && "Import completed!"}+ {importProgress?.status === "failed" && "Import failed"}+ </AlertDialogDescription>+ </AlertDialogHeader>++ <div className="space-y-4">+ {/* Progress Stats */}+ <div className="grid grid-cols-2 gap-4">+ <div>+ <p className="text-sm text-muted-foreground">Parsed Rows</p>+ <p className="text-2xl font-bold">{importProgress?.parsedRows.toLocaleString()}</p>+ </div>+ <div>+ <p className="text-sm text-muted-foreground">Skipped Rows</p>+ <p className="text-2xl font-bold">{importProgress?.skippedRows.toLocaleString()}</p>+ </div>+ <div>+ <p className="text-sm text-muted-foreground">Uploaded Batches</p>+ <p className="text-2xl font-bold">+ {importProgress?.uploadedBatches} / {importProgress?.totalBatches}+ </p>+ </div>+ <div>+ <p className="text-sm text-muted-foreground">Imported Events</p>+ <p className="text-2xl font-bold">{importProgress?.importedEvents.toLocaleString()}</p>+ </div>+ </div>++ {/* Progress Bar */}+ {importProgress && importProgress.totalBatches > 0 && (+ <div className="w-full bg-gray-200 rounded-full h-2.5">+ <div+ className="bg-blue-600 h-2.5 rounded-full transition-all"+ style={{+ width: `${(importProgress.uploadedBatches / importProgress.totalBatches) * 100}%`,+ }}+ />+ </div>+ )}++ {/* Errors */}+ {importProgress && importProgress.errors.length > 0 && (+ <div className="max-h-40 overflow-y-auto">+ <p className="text-sm font-medium text-red-600 mb-2">Errors ({importProgress.errors.length}):</p>+ {importProgress.errors.slice(0, 10).map((error, idx) => (+ <p key={idx} className="text-xs text-red-600">+ {error.batch !== undefined ? `Batch ${error.batch}: ` : ""}+ {error.message}+ </p>+ ))}+ {importProgress.errors.length > 10 && (+ <p className="text-xs text-muted-foreground">...and {importProgress.errors.length - 10} more</p>+ )}+ </div>+ )}+ </div>++ <AlertDialogFooter>+ {importProgress?.status === "completed" || importProgress?.status === "failed" ? (+ <AlertDialogAction onClick={() => setShowProgressDialog(false)}>Close</AlertDialogAction>+ ) : (+ <AlertDialogCancel+ onClick={() => {+ workerManagerRef.current?.terminate();+ setShowProgressDialog(false);+ }}+ >+ Cancel Import+ </AlertDialogCancel>+ )}+ </AlertDialogFooter>+ </AlertDialogContent>+ </AlertDialog></div>);}diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 36d6d501c..781598622 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -44,6 +44,8 @@ export class CSVWorkerManager {siteId: number,importId: string,platform: "umami",+ earliestAllowedDate: string,+ latestAllowedDate: string,startDate?: string,endDate?: string): void {@@ -89,13 +91,15 @@ export class CSVWorkerManager {}};- // Start parsing (no quota info - server handles that)+ // Start parsing with allowed date range for client-side filteringconst message: WorkerMessageToWorker = {type: "PARSE_START",file,siteId,importId,platform,+ earliestAllowedDate,+ latestAllowedDate,startDate,endDate,};diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex 292e8ddd6..03666eb97 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -1,6 +1,13 @@// Types for client-side CSV import system// Client parses CSV, server transforms and validates+// Allowed date range for imports based on subscription tier+export interface AllowedDateRange {+ earliestAllowedDate: string; // yyyy-MM-dd format+ latestAllowedDate: string; // yyyy-MM-dd format+ historicalWindowMonths: number; // 6, 24, or 60+}+export interface UmamiEvent {session_id: string;hostname: string;@@ -32,8 +39,10 @@ export type WorkerMessageToWorker =siteId: number;importId: string;platform: "umami";- startDate?: string;- endDate?: string;+ earliestAllowedDate: string; // Filter events before this date (yyyy-MM-dd)+ latestAllowedDate: string; // Filter events after this date (yyyy-MM-dd)+ startDate?: string; // Optional user-specified start date filter+ endDate?: string; // Optional user-specified end date filter}| {type: "CANCEL";diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsindex 55dd61c4b..f4c7dac2a 100644--- a/client/src/workers/csv-import.worker.ts+++ b/client/src/workers/csv-import.worker.ts@@ -17,9 +17,13 @@ let totalErrors = 0;let errorDetails: Array<{ row: number; message: string }> = [];let lastProgressUpdate = 0;-// Date range filter (client-side optimization to reduce server load)-let startDate: DateTime | null = null;-let endDate: DateTime | null = null;+// Date range filters (client-side optimization to reduce server load)+// Quota-enforced dates (required based on subscription tier)+let earliestAllowedDate: DateTime | null = null;+let latestAllowedDate: DateTime | null = null;+// User-specified optional date filters+let userStartDate: DateTime | null = null;+let userEndDate: DateTime | null = null;// Umami CSV header mappingconst umamiHeaders = [@@ -61,16 +65,38 @@ const umamiHeaders = [undefined,];-function createDateRangeFilter(startDateStr?: string, endDateStr?: string) {- startDate = startDateStr ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day") : null;- endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;+function createDateRangeFilter(+ earliestAllowedDateStr: string,+ latestAllowedDateStr: string,+ userStartDateStr?: string,+ userEndDateStr?: string+) {+ // Parse quota-enforced dates (required)+ earliestAllowedDate = DateTime.fromFormat(earliestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day");+ latestAllowedDate = DateTime.fromFormat(latestAllowedDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day");- if (startDate && !startDate.isValid) {- throw new Error(`Invalid start date: ${startDateStr}`);+ if (!earliestAllowedDate.isValid) {+ throw new Error(`Invalid earliest allowed date: ${earliestAllowedDateStr}`);}- if (endDate && !endDate.isValid) {- throw new Error(`Invalid end date: ${endDateStr}`);+ if (!latestAllowedDate.isValid) {+ throw new Error(`Invalid latest allowed date: ${latestAllowedDateStr}`);+ }++ // Parse user-specified optional dates+ userStartDate = userStartDateStr+ ? DateTime.fromFormat(userStartDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")+ : null;+ userEndDate = userEndDateStr+ ? DateTime.fromFormat(userEndDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day")+ : null;++ if (userStartDate && !userStartDate.isValid) {+ throw new Error(`Invalid user start date: ${userStartDateStr}`);+ }++ if (userEndDate && !userEndDate.isValid) {+ throw new Error(`Invalid user end date: ${userEndDateStr}`);}}@@ -80,11 +106,21 @@ function isDateInRange(dateStr: string): boolean {return false;}- if (startDate && createdAt < startDate) {+ // Check quota-enforced dates (required)+ if (earliestAllowedDate && createdAt < earliestAllowedDate) {return false;}- if (endDate && createdAt > endDate) {+ if (latestAllowedDate && createdAt > latestAllowedDate) {+ return false;+ }++ // Check user-specified optional dates+ if (userStartDate && createdAt < userStartDate) {+ return false;+ }++ if (userEndDate && createdAt > userEndDate) {return false;}@@ -123,12 +159,11 @@ function handleParsedRow(row: unknown, rowIndex: number) {return;}- // Apply optional date range filter (client-side optimization)- if (startDate || endDate) {- if (!isDateInRange(umamiEvent.created_at)) {- totalSkipped++;- return;- }+ // Apply date range filter (quota-enforced + optional user filters)+ // This client-side filtering reduces network traffic and server processing+ if (!isDateInRange(umamiEvent.created_at)) {+ totalSkipped++;+ return;}// Add to batch (no transformation, send raw row to server)@@ -218,8 +253,13 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {errorDetails = [];lastProgressUpdate = 0;- // Set up date range filter (optional client-side optimization)- createDateRangeFilter(message.startDate, message.endDate);+ // Set up date range filters (quota-enforced + optional user filters)+ createDateRangeFilter(+ message.earliestAllowedDate,+ message.latestAllowedDate,+ message.startDate,+ message.endDate+ );// Start parsingparseCSV(message.file);diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.tsnew file mode 100644index 000000000..62d6d7001--- /dev/null+++ b/server/src/api/sites/createSiteImport.ts@@ -0,0 +1,98 @@+import { FastifyReply, FastifyRequest } from "fastify";+import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";+import { ImportLimiter } from "../../services/import/importLimiter.js";+import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";+import { db } from "../../db/postgres/postgres.js";+import { sites } from "../../db/postgres/schema.js";+import { eq } from "drizzle-orm";+import { DateTime } from "luxon";+import { z } from "zod";++const createSiteImportRequestSchema = z+ .object({+ params: z.object({+ site: z.string().min(1),+ }),+ body: z.object({+ platform: z.enum(["umami"]),+ fileName: z.string().min(1),+ }),+ })+ .strict();++type CreateSiteImportRequest = {+ Params: z.infer<typeof createSiteImportRequestSchema.shape.params>;+ Body: z.infer<typeof createSiteImportRequestSchema.shape.body>;+};++export async function createSiteImport(request: FastifyRequest<CreateSiteImportRequest>, reply: FastifyReply) {+ try {+ const parsed = createSiteImportRequestSchema.safeParse({+ params: request.params,+ body: request.body,+ });++ if (!parsed.success) {+ return reply.status(400).send({ error: "Validation error", details: parsed.error });+ }++ const { site } = parsed.data.params;+ const { platform, fileName } = parsed.data.body;+ const siteId = Number(site);++ // Check user authorization+ const userHasAccess = await getUserHasAdminAccessToSite(request, site);+ if (!userHasAccess) {+ return reply.status(403).send({ error: "Forbidden" });+ }++ // Get site's organization ID+ const [siteRecord] = await db+ .select({ organizationId: sites.organizationId })+ .from(sites)+ .where(eq(sites.siteId, siteId))+ .limit(1);++ if (!siteRecord) {+ return reply.status(404).send({ error: "Site not found" });+ }++ // Create import record with concurrency check+ const importResult = await ImportLimiter.createImportWithConcurrencyCheck({+ siteId,+ organizationId: siteRecord.organizationId,+ platform,+ fileName,+ status: "pending",+ importedEvents: 0,+ errorMessage: null,+ });++ if (!importResult.success) {+ return reply.status(429).send({ error: importResult.reason });+ }++ // Get quota information to determine allowed date ranges+ const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);+ const summary = quotaTracker.getSummary();++ // Calculate the earliest and latest allowed dates+ const oldestAllowedDate = DateTime.fromFormat(summary.oldestAllowedMonth + "01", "yyyyMMdd", { zone: "utc" });+ const earliestAllowedDate = oldestAllowedDate.toFormat("yyyy-MM-dd");+ const latestAllowedDate = DateTime.utc().toFormat("yyyy-MM-dd");++ return reply.send({+ data: {+ importId: importResult.importId,+ allowedDateRange: {+ earliestAllowedDate,+ latestAllowedDate,+ historicalWindowMonths: summary.totalMonthsInWindow,+ },+ },+ });+ } catch (error) {+ console.error("Error creating import:", error);+ return reply.status(500).send({ error: "Internal server error" });+ }+}diff --git a/server/src/index.ts b/server/src/index.tsindex fcd92ffa6..5eb9b7d8c 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -88,6 +88,7 @@ import { selectGSCProperty } from "./api/gsc/selectProperty.js";import { getSiteImports } from "./api/sites/getSiteImports.js";import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";import { batchImportEvents } from "./api/sites/batchImportEvents.js";+import { createSiteImport } from "./api/sites/createSiteImport.js";const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);@@ -350,6 +351,7 @@ server.get("/api/session-replay/list/:site", getSessionReplays);server.get("/api/session-replay/:sessionId/:site", getSessionReplayEvents);// Imports+server.post("/api/create-site-import/:site", createSiteImport);server.post("/api/batch-import-events/:site", batchImportEvents);server.get("/api/get-site-imports/:site", getSiteImports);server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);