πΈ Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simplify
Edward Cui935f1af7ddf1
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 604a789b7..027f03902 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -5,17 +5,11 @@ import type { AllowedDateRange } from "@/lib/import/types";interface GetSiteImportsResponse {importId: string;- platform: "umami";+ platform: "umami" | null;status: "pending" | "processing" | "completed" | "failed";importedEvents: number;errorMessage: string | null;startedAt: string;- fileName: string;-}--interface CreateSiteImportParams {- platform: "umami";- fileName: string;}interface CreateSiteImportResponse {@@ -46,13 +40,12 @@ export function useCreateSiteImport(site: number) {const queryClient = useQueryClient();return useMutation({- mutationFn: async (params: CreateSiteImportParams) => {+ mutationFn: async () => {return await authedFetch<APIResponse<CreateSiteImportResponse>>(`/create-site-import/${site}`,undefined,{method: "POST",- data: params,});},diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex b7bbb5020..a05213b60 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -24,7 +24,7 @@ import { Alert, AlertDescription } from "@/components/ui/alert";import { Badge } from "@/components/ui/badge";import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";import { Separator } from "@/components/ui/separator";-import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Database, Trash2, Info } from "lucide-react";+import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Trash2, Info } from "lucide-react";import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/api/admin/import";import { SplitDateRangePicker } from "@/components/SplitDateRangePicker";import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";@@ -41,10 +41,8 @@ const MAX_FILE_SIZE = IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024; //const CONFIRM_THRESHOLD = IS_CLOUD ? 100 * 1024 * 1024 : 1024 * 1024 * 1024; // Show confirmation for files > 100 MB (cloud) or > 1 GB (self-hosted)const ALLOWED_FILE_TYPES = ["text/csv"];const ALLOWED_EXTENSIONS = [".csv"];-const PLATFORMS = [{ value: "umami", label: "Umami" }] as const;const importFormSchema = z.object({- platform: z.enum(["umami"], { required_error: "Please select a platform" }),file: z.custom<FileList>().refine(files => files.length === 1, "Please select a file")@@ -132,7 +130,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {resolver: zodResolver(importFormSchema),mode: "onChange",defaultValues: {- platform: "" as "umami",file: Object.assign(new DataTransfer().files, {}),dateRange: {},},@@ -168,49 +165,42 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const endDate = data.dateRange.endDate?.toFormat("yyyy-MM-dd");// Step 1: Create import record and get allowed date range- createImportMutation.mutate(- {- platform: data.platform,- fileName: file.name,+ createImportMutation.mutate(undefined, {+ onSuccess: response => {+ const { importId, allowedDateRange } = response.data;++ // Step 2: Initialize worker manager+ workerManagerRef.current = new CSVWorkerManager(+ progress => {+ setImportProgress(progress);+ },+ (success, message) => {+ console.log(success ? "β " : "β", message);+ // Import list will auto-refresh due to polling in useGetSiteImports+ }+ );++ // Step 3: Show progress dialog+ setShowProgressDialog(true);++ // Step 4: Start CSV parsing and upload with allowed date range+ workerManagerRef.current.startImport(+ file,+ siteId,+ importId,+ allowedDateRange.earliestAllowedDate,+ allowedDateRange.latestAllowedDate,+ startDate,+ endDate+ );++ // Reset form+ reset();},- {- onSuccess: response => {- const { importId, allowedDateRange } = response.data;-- // Step 2: Initialize worker manager- workerManagerRef.current = new CSVWorkerManager(- progress => {- setImportProgress(progress);- },- (success, message) => {- console.log(success ? "β " : "β", message);- // Import list will auto-refresh due to polling in useGetSiteImports- }- );-- // Step 3: Show progress dialog- setShowProgressDialog(true);-- // Step 4: Start CSV parsing and upload with allowed date range- workerManagerRef.current.startImport(- file,- siteId,- importId,- data.platform,- allowedDateRange.earliestAllowedDate,- allowedDateRange.latestAllowedDate,- startDate,- endDate- );-- // Reset form- reset();- },- onError: error => {- console.error("Failed to create import:", error);- },- }- );+ onError: error => {+ console.error("Failed to create import:", error);+ },+ });setShowConfirmDialog(false);};@@ -313,37 +303,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {)}<form onSubmit={handleSubmit(onSubmit)} className="space-y-4">- {/* Platform Selection */}- <div className="space-y-2">- <Label htmlFor="platform" className="flex items-center gap-2">- <Database className="h-4 w-4" />- Platform- </Label>- <Controller- name="platform"- control={control}- render={({ field }) => (- <Select- value={field.value}- onValueChange={field.onChange}- disabled={disabled || createImportMutation.isPending || hasActiveImport}- >- <SelectTrigger>- <SelectValue placeholder="Select platform" />- </SelectTrigger>- <SelectContent>- {PLATFORMS.map(platform => (- <SelectItem key={platform.value} value={platform.value}>- {platform.label}- </SelectItem>- ))}- </SelectContent>- </Select>- )}- />- {errors.platform && <p className="text-sm text-red-600">{errors.platform.message}</p>}- </div>-{/* Date Range Picker */}<Controllername="dateRange"@@ -477,7 +436,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Table><TableHeader><TableRow>- <TableHead>File</TableHead>+ <TableHead>Started At</TableHead><TableHead>Platform</TableHead><TableHead>Status</TableHead><TableHead className="text-right">Events</TableHead>@@ -488,17 +447,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {{sortedImports.map(imp => {const statusInfo = getStatusInfo(imp.status);const StatusIcon = statusInfo.icon;+ const startedAt = DateTime.fromISO(imp.startedAt).toFormat("MMM dd, yyyy HH:mm");return (<TableRow key={imp.importId}>- <TableCell className="font-medium">- <div className="max-w-[200px] truncate" title={imp.fileName}>- {imp.fileName}- </div>- </TableCell>+ <TableCell className="font-medium">{startedAt}</TableCell><TableCell><Badge variant="outline" className="capitalize">- {imp.platform}+ {imp.platform || "N/A"}</Badge></TableCell><TableCell>diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex a7dbd15ad..4b0ea066a 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -43,7 +43,6 @@ export class CSVWorkerManager {file: File,siteId: number,importId: string,- platform: "umami",earliestAllowedDate: string,latestAllowedDate: string,startDate?: string,@@ -97,7 +96,6 @@ export class CSVWorkerManager {file,siteId,importId,- platform,earliestAllowedDate,latestAllowedDate,startDate,diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex 03666eb97..0ee73a8ed 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -38,7 +38,6 @@ export type WorkerMessageToWorker =file: File;siteId: number;importId: string;- platform: "umami";earliestAllowedDate: string; // Filter events before this date (yyyy-MM-dd)latestAllowedDate: string; // Filter events after this date (yyyy-MM-dd)startDate?: string; // Optional user-specified start date filterdiff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsindex 22319bfa9..244e8637a 100644--- a/client/src/workers/csv-import.worker.ts+++ b/client/src/workers/csv-import.worker.ts@@ -151,7 +151,32 @@ function sendProgress() {}function handleParsedRow(row: unknown, rowIndex: number) {- const umamiEvent = row as UmamiEvent;+ const rawEvent = row as Record<string, unknown>;++ // Filter out undefined columns (those mapped to undefined in umamiHeaders)+ // This reduces payload size by removing unused CSV columns+ const umamiEvent: UmamiEvent = {+ session_id: String(rawEvent.session_id || ""),+ hostname: String(rawEvent.hostname || ""),+ browser: String(rawEvent.browser || ""),+ os: String(rawEvent.os || ""),+ device: String(rawEvent.device || ""),+ screen: String(rawEvent.screen || ""),+ language: String(rawEvent.language || ""),+ country: String(rawEvent.country || ""),+ region: String(rawEvent.region || ""),+ city: String(rawEvent.city || ""),+ url_path: String(rawEvent.url_path || ""),+ url_query: String(rawEvent.url_query || ""),+ referrer_path: String(rawEvent.referrer_path || ""),+ referrer_query: String(rawEvent.referrer_query || ""),+ referrer_domain: String(rawEvent.referrer_domain || ""),+ page_title: String(rawEvent.page_title || ""),+ event_type: String(rawEvent.event_type || ""),+ event_name: String(rawEvent.event_name || ""),+ distinct_id: String(rawEvent.distinct_id || ""),+ created_at: String(rawEvent.created_at || ""),+ };// Skip rows with missing created_at (required field)if (!umamiEvent.created_at) {@@ -166,7 +191,7 @@ function handleParsedRow(row: unknown, rowIndex: number) {return;}- // Add to batch (no transformation, send raw row to server)+ // Add to batch (filtered event with only required fields)currentBatch.push(umamiEvent);totalParsed++;diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 264da7b41..ba80ed3fb 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -6,19 +6,43 @@ import { updateImportProgress, updateImportStatus, getImportById } from "../../simport { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";import { db } from "../../db/postgres/postgres.js";-import { sites } from "../../db/postgres/schema.js";+import { sites, importStatus } from "../../db/postgres/schema.js";import { eq } from "drizzle-orm";import { createServiceLogger } from "../../lib/logger/logger.js";const logger = createServiceLogger("import:batch");+// Zod schema for Umami event (from CSV)+const umamiEventSchema = z.object({+ session_id: z.string(),+ hostname: z.string(),+ browser: z.string(),+ os: z.string(),+ device: z.string(),+ screen: z.string(),+ language: z.string(),+ country: z.string(),+ region: z.string(),+ city: z.string(),+ url_path: z.string(),+ url_query: z.string(),+ referrer_path: z.string(),+ referrer_query: z.string(),+ referrer_domain: z.string(),+ page_title: z.string(),+ event_type: z.string(),+ event_name: z.string(),+ distinct_id: z.string(),+ created_at: z.string(),+});+const batchImportRequestSchema = z.object({params: z.object({site: z.string().min(1),}),body: z.object({- events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events, raw Umami rows+ events: z.array(umamiEventSchema).min(1).max(10000), // Properly typed Umami eventsimportId: z.string().uuid(),batchIndex: z.number().int().min(0),totalBatches: z.number().int().min(1),@@ -79,6 +103,22 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeawait updateImportStatus(importId, "processing");}+ // Auto-detect platform if not set (first batch)+ let detectedPlatform = importRecord.platform;+ if (!detectedPlatform) {+ // Detect platform based on event structure+ // For now, we only support Umami, so if events match Umami schema, it's Umami+ detectedPlatform = "umami";++ // Update import record with detected platform+ await db+ .update(importStatus)+ .set({ platform: detectedPlatform })+ .where(eq(importStatus.importId, importId));++ logger.info({ importId, detectedPlatform }, "Auto-detected platform");+ }+// Get organization ID for quota checkingconst [siteRecord] = await db.select({ organizationId: sites.organizationId })@@ -99,7 +139,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeconst eventsWithinQuota: UmamiEvent[] = [];let skippedDueToQuota = 0;- for (const event of events as UmamiEvent[]) {+ for (const event of events) {if (!event.created_at) {continue; // Skip events without timestamp}diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.tsindex 62d6d7001..4e5179ee6 100644--- a/server/src/api/sites/createSiteImport.ts+++ b/server/src/api/sites/createSiteImport.ts@@ -13,23 +13,17 @@ const createSiteImportRequestSchema = zparams: z.object({site: z.string().min(1),}),- body: z.object({- platform: z.enum(["umami"]),- fileName: z.string().min(1),- }),}).strict();type CreateSiteImportRequest = {Params: z.infer<typeof createSiteImportRequestSchema.shape.params>;- Body: z.infer<typeof createSiteImportRequestSchema.shape.body>;};export async function createSiteImport(request: FastifyRequest<CreateSiteImportRequest>, reply: FastifyReply) {try {const parsed = createSiteImportRequestSchema.safeParse({params: request.params,- body: request.body,});if (!parsed.success) {@@ -37,7 +31,6 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportR}const { site } = parsed.data.params;- const { platform, fileName } = parsed.data.body;const siteId = Number(site);// Check user authorization@@ -57,12 +50,11 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportRreturn reply.status(404).send({ error: "Site not found" });}- // Create import record with concurrency check+ // Create import record with concurrency check (platform will be auto-detected on first batch)const importResult = await ImportLimiter.createImportWithConcurrencyCheck({siteId,organizationId: siteRecord.organizationId,- platform,- fileName,+ platform: null,status: "pending",importedEvents: 0,errorMessage: null,diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 1e1411877..2e657a664 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -632,13 +632,12 @@ export const importStatus = pgTable(importId: uuid("import_id").primaryKey().notNull().defaultRandom(),siteId: integer("site_id").notNull(),organizationId: text("organization_id").notNull(),- platform: importPlatformEnum("platform").notNull(),+ platform: importPlatformEnum("platform"),status: importStatusEnum("status").notNull().default("pending"),importedEvents: integer("imported_events").notNull().default(0),errorMessage: text("error_message"),startedAt: timestamp("started_at", { mode: "string" }).notNull().defaultNow(),completedAt: timestamp("completed_at", { mode: "string" }),- fileName: text("file_name").notNull(),},table => [foreignKey({