Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simplify

Edward Cui935f1af7ddf1
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 604a789b7..027f03902 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -5,17 +5,11 @@ import type { AllowedDateRange } from "@/lib/import/types";
interface GetSiteImportsResponse {
importId: string;
- platform: "umami";
+ platform: "umami" | null;
status: "pending" | "processing" | "completed" | "failed";
importedEvents: number;
errorMessage: string | null;
startedAt: string;
- fileName: string;
-}
-
-interface CreateSiteImportParams {
- platform: "umami";
- fileName: string;
}
interface CreateSiteImportResponse {
@@ -46,13 +40,12 @@ export function useCreateSiteImport(site: number) {
const queryClient = useQueryClient();
return useMutation({
- mutationFn: async (params: CreateSiteImportParams) => {
+ mutationFn: async () => {
return await authedFetch<APIResponse<CreateSiteImportResponse>>(
`/create-site-import/${site}`,
undefined,
{
method: "POST",
- data: params,
}
);
},
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index b7bbb5020..a05213b60 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -24,7 +24,7 @@ import { Alert, AlertDescription } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Separator } from "@/components/ui/separator";
-import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Database, Trash2, Info } from "lucide-react";
+import { Upload, FileText, AlertCircle, CheckCircle2, Clock, Loader2, Trash2, Info } from "lucide-react";
import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/api/admin/import";
import { SplitDateRangePicker } from "@/components/SplitDateRangePicker";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
@@ -41,10 +41,8 @@ const MAX_FILE_SIZE = IS_CLOUD ? 500 * 1024 * 1024 : 50 * 1024 * 1024 * 1024; //
const CONFIRM_THRESHOLD = IS_CLOUD ? 100 * 1024 * 1024 : 1024 * 1024 * 1024; // Show confirmation for files > 100 MB (cloud) or > 1 GB (self-hosted)
const ALLOWED_FILE_TYPES = ["text/csv"];
const ALLOWED_EXTENSIONS = [".csv"];
-const PLATFORMS = [{ value: "umami", label: "Umami" }] as const;
const importFormSchema = z.object({
- platform: z.enum(["umami"], { required_error: "Please select a platform" }),
file: z
.custom<FileList>()
.refine(files => files.length === 1, "Please select a file")
@@ -132,7 +130,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
resolver: zodResolver(importFormSchema),
mode: "onChange",
defaultValues: {
- platform: "" as "umami",
file: Object.assign(new DataTransfer().files, {}),
dateRange: {},
},
@@ -168,49 +165,42 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const endDate = data.dateRange.endDate?.toFormat("yyyy-MM-dd");
// Step 1: Create import record and get allowed date range
- createImportMutation.mutate(
- {
- platform: data.platform,
- fileName: file.name,
+ createImportMutation.mutate(undefined, {
+ onSuccess: response => {
+ const { importId, allowedDateRange } = response.data;
+
+ // Step 2: Initialize worker manager
+ workerManagerRef.current = new CSVWorkerManager(
+ progress => {
+ setImportProgress(progress);
+ },
+ (success, message) => {
+ console.log(success ? "βœ…" : "❌", message);
+ // Import list will auto-refresh due to polling in useGetSiteImports
+ }
+ );
+
+ // Step 3: Show progress dialog
+ setShowProgressDialog(true);
+
+ // Step 4: Start CSV parsing and upload with allowed date range
+ workerManagerRef.current.startImport(
+ file,
+ siteId,
+ importId,
+ allowedDateRange.earliestAllowedDate,
+ allowedDateRange.latestAllowedDate,
+ startDate,
+ endDate
+ );
+
+ // Reset form
+ reset();
},
- {
- onSuccess: response => {
- const { importId, allowedDateRange } = response.data;
-
- // Step 2: Initialize worker manager
- workerManagerRef.current = new CSVWorkerManager(
- progress => {
- setImportProgress(progress);
- },
- (success, message) => {
- console.log(success ? "βœ…" : "❌", message);
- // Import list will auto-refresh due to polling in useGetSiteImports
- }
- );
-
- // Step 3: Show progress dialog
- setShowProgressDialog(true);
-
- // Step 4: Start CSV parsing and upload with allowed date range
- workerManagerRef.current.startImport(
- file,
- siteId,
- importId,
- data.platform,
- allowedDateRange.earliestAllowedDate,
- allowedDateRange.latestAllowedDate,
- startDate,
- endDate
- );
-
- // Reset form
- reset();
- },
- onError: error => {
- console.error("Failed to create import:", error);
- },
- }
- );
+ onError: error => {
+ console.error("Failed to create import:", error);
+ },
+ });
setShowConfirmDialog(false);
};
@@ -313,37 +303,6 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
)}
<form onSubmit={handleSubmit(onSubmit)} className="space-y-4">
- {/* Platform Selection */}
- <div className="space-y-2">
- <Label htmlFor="platform" className="flex items-center gap-2">
- <Database className="h-4 w-4" />
- Platform
- </Label>
- <Controller
- name="platform"
- control={control}
- render={({ field }) => (
- <Select
- value={field.value}
- onValueChange={field.onChange}
- disabled={disabled || createImportMutation.isPending || hasActiveImport}
- >
- <SelectTrigger>
- <SelectValue placeholder="Select platform" />
- </SelectTrigger>
- <SelectContent>
- {PLATFORMS.map(platform => (
- <SelectItem key={platform.value} value={platform.value}>
- {platform.label}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
- )}
- />
- {errors.platform && <p className="text-sm text-red-600">{errors.platform.message}</p>}
- </div>
-
{/* Date Range Picker */}
<Controller
name="dateRange"
@@ -477,7 +436,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Table>
<TableHeader>
<TableRow>
- <TableHead>File</TableHead>
+ <TableHead>Started At</TableHead>
<TableHead>Platform</TableHead>
<TableHead>Status</TableHead>
<TableHead className="text-right">Events</TableHead>
@@ -488,17 +447,14 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
{sortedImports.map(imp => {
const statusInfo = getStatusInfo(imp.status);
const StatusIcon = statusInfo.icon;
+ const startedAt = DateTime.fromISO(imp.startedAt).toFormat("MMM dd, yyyy HH:mm");
return (
<TableRow key={imp.importId}>
- <TableCell className="font-medium">
- <div className="max-w-[200px] truncate" title={imp.fileName}>
- {imp.fileName}
- </div>
- </TableCell>
+ <TableCell className="font-medium">{startedAt}</TableCell>
<TableCell>
<Badge variant="outline" className="capitalize">
- {imp.platform}
+ {imp.platform || "N/A"}
</Badge>
</TableCell>
<TableCell>
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index a7dbd15ad..4b0ea066a 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -43,7 +43,6 @@ export class CSVWorkerManager {
file: File,
siteId: number,
importId: string,
- platform: "umami",
earliestAllowedDate: string,
latestAllowedDate: string,
startDate?: string,
@@ -97,7 +96,6 @@ export class CSVWorkerManager {
file,
siteId,
importId,
- platform,
earliestAllowedDate,
latestAllowedDate,
startDate,
diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.ts
index 03666eb97..0ee73a8ed 100644
--- a/client/src/lib/import/types.ts
+++ b/client/src/lib/import/types.ts
@@ -38,7 +38,6 @@ export type WorkerMessageToWorker =
file: File;
siteId: number;
importId: string;
- platform: "umami";
earliestAllowedDate: string; // Filter events before this date (yyyy-MM-dd)
latestAllowedDate: string; // Filter events after this date (yyyy-MM-dd)
startDate?: string; // Optional user-specified start date filter
diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.ts
index 22319bfa9..244e8637a 100644
--- a/client/src/workers/csv-import.worker.ts
+++ b/client/src/workers/csv-import.worker.ts
@@ -151,7 +151,32 @@ function sendProgress() {
}
function handleParsedRow(row: unknown, rowIndex: number) {
- const umamiEvent = row as UmamiEvent;
+ const rawEvent = row as Record<string, unknown>;
+
+ // Filter out undefined columns (those mapped to undefined in umamiHeaders)
+ // This reduces payload size by removing unused CSV columns
+ const umamiEvent: UmamiEvent = {
+ session_id: String(rawEvent.session_id || ""),
+ hostname: String(rawEvent.hostname || ""),
+ browser: String(rawEvent.browser || ""),
+ os: String(rawEvent.os || ""),
+ device: String(rawEvent.device || ""),
+ screen: String(rawEvent.screen || ""),
+ language: String(rawEvent.language || ""),
+ country: String(rawEvent.country || ""),
+ region: String(rawEvent.region || ""),
+ city: String(rawEvent.city || ""),
+ url_path: String(rawEvent.url_path || ""),
+ url_query: String(rawEvent.url_query || ""),
+ referrer_path: String(rawEvent.referrer_path || ""),
+ referrer_query: String(rawEvent.referrer_query || ""),
+ referrer_domain: String(rawEvent.referrer_domain || ""),
+ page_title: String(rawEvent.page_title || ""),
+ event_type: String(rawEvent.event_type || ""),
+ event_name: String(rawEvent.event_name || ""),
+ distinct_id: String(rawEvent.distinct_id || ""),
+ created_at: String(rawEvent.created_at || ""),
+ };
// Skip rows with missing created_at (required field)
if (!umamiEvent.created_at) {
@@ -166,7 +191,7 @@ function handleParsedRow(row: unknown, rowIndex: number) {
return;
}
- // Add to batch (no transformation, send raw row to server)
+ // Add to batch (filtered event with only required fields)
currentBatch.push(umamiEvent);
totalParsed++;
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 264da7b41..ba80ed3fb 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -6,19 +6,43 @@ import { updateImportProgress, updateImportStatus, getImportById } from "../../s
import { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";
import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
+import { sites, importStatus } from "../../db/postgres/schema.js";
import { eq } from "drizzle-orm";
import { createServiceLogger } from "../../lib/logger/logger.js";
const logger = createServiceLogger("import:batch");
+// Zod schema for Umami event (from CSV)
+const umamiEventSchema = z.object({
+ session_id: z.string(),
+ hostname: z.string(),
+ browser: z.string(),
+ os: z.string(),
+ device: z.string(),
+ screen: z.string(),
+ language: z.string(),
+ country: z.string(),
+ region: z.string(),
+ city: z.string(),
+ url_path: z.string(),
+ url_query: z.string(),
+ referrer_path: z.string(),
+ referrer_query: z.string(),
+ referrer_domain: z.string(),
+ page_title: z.string(),
+ event_type: z.string(),
+ event_name: z.string(),
+ distinct_id: z.string(),
+ created_at: z.string(),
+});
+
const batchImportRequestSchema = z
.object({
params: z.object({
site: z.string().min(1),
}),
body: z.object({
- events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events, raw Umami rows
+ events: z.array(umamiEventSchema).min(1).max(10000), // Properly typed Umami events
importId: z.string().uuid(),
batchIndex: z.number().int().min(0),
totalBatches: z.number().int().min(1),
@@ -79,6 +103,22 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
await updateImportStatus(importId, "processing");
}
+ // Auto-detect platform if not set (first batch)
+ let detectedPlatform = importRecord.platform;
+ if (!detectedPlatform) {
+ // Detect platform based on event structure
+ // For now, we only support Umami, so if events match Umami schema, it's Umami
+ detectedPlatform = "umami";
+
+ // Update import record with detected platform
+ await db
+ .update(importStatus)
+ .set({ platform: detectedPlatform })
+ .where(eq(importStatus.importId, importId));
+
+ logger.info({ importId, detectedPlatform }, "Auto-detected platform");
+ }
+
// Get organization ID for quota checking
const [siteRecord] = await db
.select({ organizationId: sites.organizationId })
@@ -99,7 +139,7 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
const eventsWithinQuota: UmamiEvent[] = [];
let skippedDueToQuota = 0;
- for (const event of events as UmamiEvent[]) {
+ for (const event of events) {
if (!event.created_at) {
continue; // Skip events without timestamp
}
diff --git a/server/src/api/sites/createSiteImport.ts b/server/src/api/sites/createSiteImport.ts
index 62d6d7001..4e5179ee6 100644
--- a/server/src/api/sites/createSiteImport.ts
+++ b/server/src/api/sites/createSiteImport.ts
@@ -13,23 +13,17 @@ const createSiteImportRequestSchema = z
params: z.object({
site: z.string().min(1),
}),
- body: z.object({
- platform: z.enum(["umami"]),
- fileName: z.string().min(1),
- }),
})
.strict();
type CreateSiteImportRequest = {
Params: z.infer<typeof createSiteImportRequestSchema.shape.params>;
- Body: z.infer<typeof createSiteImportRequestSchema.shape.body>;
};
export async function createSiteImport(request: FastifyRequest<CreateSiteImportRequest>, reply: FastifyReply) {
try {
const parsed = createSiteImportRequestSchema.safeParse({
params: request.params,
- body: request.body,
});
if (!parsed.success) {
@@ -37,7 +31,6 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportR
}
const { site } = parsed.data.params;
- const { platform, fileName } = parsed.data.body;
const siteId = Number(site);
// Check user authorization
@@ -57,12 +50,11 @@ export async function createSiteImport(request: FastifyRequest<CreateSiteImportR
return reply.status(404).send({ error: "Site not found" });
}
- // Create import record with concurrency check
+ // Create import record with concurrency check (platform will be auto-detected on first batch)
const importResult = await ImportLimiter.createImportWithConcurrencyCheck({
siteId,
organizationId: siteRecord.organizationId,
- platform,
- fileName,
+ platform: null,
status: "pending",
importedEvents: 0,
errorMessage: null,
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 1e1411877..2e657a664 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -632,13 +632,12 @@ export const importStatus = pgTable(
importId: uuid("import_id").primaryKey().notNull().defaultRandom(),
siteId: integer("site_id").notNull(),
organizationId: text("organization_id").notNull(),
- platform: importPlatformEnum("platform").notNull(),
+ platform: importPlatformEnum("platform"),
status: importStatusEnum("status").notNull().default("pending"),
importedEvents: integer("imported_events").notNull().default(0),
errorMessage: text("error_message"),
startedAt: timestamp("started_at", { mode: "string" }).notNull().defaultNow(),
completedAt: timestamp("completed_at", { mode: "string" }),
- fileName: text("file_name").notNull(),
},
table => [
foreignKey({