🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Simple Analytics import (#741)
Edward Cui185e1f5abe85
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 282c445a8..1d4585b50 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -1,10 +1,11 @@import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";import { authedFetch } from "@/api/utils";import { APIResponse } from "@/api/types";+import { ImportPlatform } from "@/types/import";interface GetSiteImportsResponse {importId: string;- platform: "umami";+ platform: ImportPlatform;importedEvents: number;skippedEvents: number;invalidEvents: number;@@ -37,7 +38,7 @@ export function useCreateSiteImport(site: number) {const queryClient = useQueryClient();return useMutation({- mutationFn: async (data: { platform: "umami" }) => {+ mutationFn: async (data: { platform: ImportPlatform }) => {return await authedFetch<APIResponse<CreateSiteImportResponse>>(`/create-site-import/${site}`, undefined, {method: "POST",data,diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex bcfb36e9c..fdb87a337 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -25,6 +25,7 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/aimport { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";import { IS_CLOUD } from "@/lib/const";import { CsvParser } from "@/lib/import/csvParser";+import { ImportPlatform } from "@/types/import";interface ImportManagerProps {siteId: number;@@ -59,11 +60,19 @@ function formatFileSize(bytes: number): string {}}+function formatPlatformName(platform: ImportPlatform): string {+ const platformNames: Record<ImportPlatform, string> = {+ umami: "Umami",+ simple_analytics: "Simple Analytics",+ };+ return platformNames[platform];+}+export function ImportManager({ siteId, disabled }: ImportManagerProps) {const [showConfirmDialog, setShowConfirmDialog] = useState(false);const [importToDelete, setImportToDelete] = useState<string | null>(null);const [selectedFile, setSelectedFile] = useState<File | null>(null);- const [selectedPlatform, setSelectedPlatform] = useState<"umami" | "">("");+ const [selectedPlatform, setSelectedPlatform] = useState<ImportPlatform | "">("");const [fileError, setFileError] = useState<string>("");const fileInputRef = useRef<HTMLInputElement>(null);const workerManagerRef = useRef<CsvParser | null>(null);@@ -104,10 +113,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {onSuccess: response => {const { importId, allowedDateRange } = response.data;- workerManagerRef.current = new CsvParser();-- workerManagerRef.current.startImport(- selectedFile,+ workerManagerRef.current = new CsvParser(siteId,importId,selectedPlatform,@@ -115,6 +121,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {allowedDateRange.latestAllowedDate);+ workerManagerRef.current.startImport(selectedFile);+setSelectedFile(null);setSelectedPlatform("");setFileError("");@@ -204,12 +212,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {{/* Platform Selection */}<div className="space-y-2"><Label htmlFor="platform">Platform</Label>- <Select value={selectedPlatform} onValueChange={(value: "umami") => setSelectedPlatform(value)}>+ <Select value={selectedPlatform} onValueChange={(value: ImportPlatform) => setSelectedPlatform(value)}><SelectTrigger id="platform" disabled={disabled || createImportMutation.isPending || hasActiveImport}><SelectValue placeholder="Select platform" /></SelectTrigger><SelectContent><SelectItem value="umami">Umami</SelectItem>+ <SelectItem value="simple_analytics">Simple Analytics</SelectItem></SelectContent></Select></div>@@ -321,9 +330,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<TableRow key={imp.importId}><TableCell className="font-medium">{startedAt}</TableCell><TableCell>- <Badge variant="outline" className="capitalize">- {imp.platform}- </Badge>+ <Badge variant="outline">{formatPlatformName(imp.platform)}</Badge></TableCell><TableCell><Badge className={`${statusInfo.color} flex items-center gap-1`}>diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.tsindex 0b98d3bc8..839de4190 100644--- a/client/src/lib/import/csvParser.ts+++ b/client/src/lib/import/csvParser.ts@@ -1,6 +1,7 @@import Papa from "papaparse";import { DateTime } from "luxon";import { authedFetch } from "@/api/utils";+import { ImportPlatform } from "@/types/import";interface UmamiEvent {session_id: string;@@ -24,37 +25,52 @@ interface UmamiEvent {created_at: string;}-export class CsvParser {- private cancelled = false;- private siteId: number = 0;- private importId: string = "";- private platform: "umami" = "umami";-- private earliestAllowedDate: DateTime | null = null;- private latestAllowedDate: DateTime | null = null;+interface SimpleAnalyticsEvent {+ added_iso: string;+ country_code: string;+ datapoint: string;+ document_referrer: string;+ hostname: string;+ lang_language: string;+ lang_region: string;+ path: string;+ query: string;+ screen_height: string;+ screen_width: string;+ session_id: string;+ user_agent: string;+ uuid: string;+}- startImport(- file: File,+export class CsvParser {+ private cancelled: boolean = false;+ private readonly siteId: number;+ private readonly importId: string;+ private readonly platform: ImportPlatform;+ private readonly earliestAllowedDate: DateTime;+ private readonly latestAllowedDate: DateTime;++ constructor(siteId: number,importId: string,- platform: "umami",+ platform: ImportPlatform,earliestAllowedDate: string,latestAllowedDate: string- ): void {+ ) {this.siteId = siteId;this.importId = importId;this.platform = platform;-this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");- if (!this.earliestAllowedDate.isValid) {+ // Pre-validate dates during instantiation+ if (!this.earliestAllowedDate.isValid || !this.latestAllowedDate.isValid) {this.cancelled = true;- return;}+ }- if (!this.latestAllowedDate.isValid) {- this.cancelled = true;+ startImport(file: File): void {+ if (this.cancelled) {return;}@@ -70,16 +86,28 @@ export class CsvParser {}try {- const validEvents: UmamiEvent[] = [];- for (const row of results.data) {- const event = this.transformRow(row);- if (event && this.isDateInRange(event.created_at)) {- validEvents.push(event);+ if (this.platform === "umami") {+ const validEvents: UmamiEvent[] = [];+ for (const row of results.data) {+ const event = this.transformRow(row);+ if (event && this.isDateInRange((event as UmamiEvent).created_at)) {+ validEvents.push(event as UmamiEvent);+ }+ }+ if (validEvents.length > 0) {+ await this.uploadChunk(validEvents, false);+ }+ } else {+ const validEvents: SimpleAnalyticsEvent[] = [];+ for (const row of results.data) {+ const event = this.transformRow(row);+ if (event && this.isDateInRange((event as SimpleAnalyticsEvent).added_iso)) {+ validEvents.push(event as SimpleAnalyticsEvent);+ }+ }+ if (validEvents.length > 0) {+ await this.uploadChunk(validEvents, false);}- }-- if (validEvents.length > 0) {- await this.uploadChunk(validEvents, false);}} catch (error) {console.error("Error uploading chunk:", error);@@ -105,55 +133,84 @@ export class CsvParser {}private isDateInRange(dateStr: string): boolean {- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ // Handle both formats: "yyyy-MM-dd HH:mm:ss" (Umami) and ISO (Simple Analytics)+ let createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });+ if (!createdAt.isValid) {+ createdAt = DateTime.fromISO(dateStr, { zone: "utc" });+ }if (!createdAt.isValid) {return false;}- if (this.earliestAllowedDate && createdAt < this.earliestAllowedDate) {+ if (createdAt < this.earliestAllowedDate) {return false;}- if (this.latestAllowedDate && createdAt > this.latestAllowedDate) {+ if (createdAt > this.latestAllowedDate) {return false;}return true;}- private transformRow(row: unknown): UmamiEvent | null {+ private transformRow(row: unknown): UmamiEvent | SimpleAnalyticsEvent | null {const rawEvent = row as Record<string, string>;- const umamiEvent: UmamiEvent = {- session_id: rawEvent.session_id,- hostname: rawEvent.hostname,- browser: rawEvent.browser,- os: rawEvent.os,- device: rawEvent.device,- screen: rawEvent.screen,- language: rawEvent.language,- country: rawEvent.country,- region: rawEvent.region,- city: rawEvent.city,- url_path: rawEvent.url_path,- url_query: rawEvent.url_query,- referrer_path: rawEvent.referrer_path,- referrer_domain: rawEvent.referrer_domain,- page_title: rawEvent.page_title,- event_type: rawEvent.event_type,- event_name: rawEvent.event_name,- distinct_id: rawEvent.distinct_id,- created_at: rawEvent.created_at,- };-- if (!umamiEvent.created_at) {- return null;+ if (this.platform === "umami") {+ const umamiEvent: UmamiEvent = {+ session_id: rawEvent.session_id,+ hostname: rawEvent.hostname,+ browser: rawEvent.browser,+ os: rawEvent.os,+ device: rawEvent.device,+ screen: rawEvent.screen,+ language: rawEvent.language,+ country: rawEvent.country,+ region: rawEvent.region,+ city: rawEvent.city,+ url_path: rawEvent.url_path,+ url_query: rawEvent.url_query,+ referrer_path: rawEvent.referrer_path,+ referrer_domain: rawEvent.referrer_domain,+ page_title: rawEvent.page_title,+ event_type: rawEvent.event_type,+ event_name: rawEvent.event_name,+ distinct_id: rawEvent.distinct_id,+ created_at: rawEvent.created_at,+ };++ if (!umamiEvent.created_at) {+ return null;+ }++ return umamiEvent;+ } else {+ const simpleAnalyticsEvent: SimpleAnalyticsEvent = {+ added_iso: rawEvent.added_iso,+ country_code: rawEvent.country_code,+ datapoint: rawEvent.datapoint,+ document_referrer: rawEvent.document_referrer,+ hostname: rawEvent.hostname,+ lang_language: rawEvent.lang_language,+ lang_region: rawEvent.lang_region,+ path: rawEvent.path,+ query: rawEvent.query,+ screen_height: rawEvent.screen_height,+ screen_width: rawEvent.screen_width,+ session_id: rawEvent.session_id,+ user_agent: rawEvent.user_agent,+ uuid: rawEvent.uuid,+ };++ if (!simpleAnalyticsEvent.added_iso) {+ return null;+ }++ return simpleAnalyticsEvent;}-- return umamiEvent;}- private async uploadChunk(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {+ private async uploadChunk(events: UmamiEvent[] | SimpleAnalyticsEvent[], isLastBatch: boolean): Promise<void> {// Skip empty chunks unless it's the last one (needed for finalization)if (events.length === 0 && !isLastBatch) {return;diff --git a/client/src/types/import.ts b/client/src/types/import.tsnew file mode 100644index 000000000..8dde561c7--- /dev/null+++ b/client/src/types/import.ts@@ -0,0 +1 @@+export type ImportPlatform = "umami" | "simple_analytics";diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 583901247..e153f3558 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -3,7 +3,8 @@ import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";-import { UmamiImportMapper } from "../../services/import/mappings/umami.js";+import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappings/umami.js";+import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappings/simpleAnalytics.js";import { importQuotaManager } from "../../services/import/importQuotaManager.js";import { db } from "../../db/postgres/postgres.js";import { sites } from "../../db/postgres/schema.js";@@ -16,7 +17,10 @@ const batchImportRequestSchema = zimportId: z.string().uuid(),}),body: z.object({- events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema),+ events: z.union([+ z.array(UmamiImportMapper.umamiEventKeyOnlySchema),+ z.array(SimpleAnalyticsImportMapper.simpleAnalyticsEventKeyOnlySchema),+ ]),isLastBatch: z.boolean().optional(),}),})@@ -68,7 +72,18 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequetry {const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);- const transformedEvents = UmamiImportMapper.transform(events, site, importId);+ let transformedEvents;+ if (importRecord.platform === "umami") {+ transformedEvents = UmamiImportMapper.transform(events as UmamiEvent[], site, importId);+ } else if (importRecord.platform === "simple_analytics") {+ transformedEvents = SimpleAnalyticsImportMapper.transform(+ events as SimpleAnalyticsEvent[],+ site,+ importId+ );+ } else {+ return reply.status(400).send({ error: "Unsupported platform" });+ }const invalidEventCount = events.length - transformedEvents.length;const timestamps = transformedEvents.map(e => e.timestamp);diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 1cacb6fe2..ff417b03c 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -563,7 +563,7 @@ export const userAliases = pgTable(]);-export const importPlatforms = ["umami"] as const;+export const importPlatforms = ["umami", "simple_analytics"] as const;export const importPlatformEnum = pgEnum("import_platform_enum", importPlatforms);diff --git a/server/src/services/import/mappings/simpleAnalytics.ts b/server/src/services/import/mappings/simpleAnalytics.tsnew file mode 100644index 000000000..1e29a06d2--- /dev/null+++ b/server/src/services/import/mappings/simpleAnalytics.ts@@ -0,0 +1,101 @@+import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";+import { getChannel } from "../../tracker/getChannel.js";+import { RybbitEvent } from "./rybbit.js";+import { z } from "zod";+import { UAParser } from "ua-parser-js";+import { DateTime } from "luxon";+import { getDeviceType } from "../../../utils.js";+import { deriveKeyOnlySchema } from "./utils.js";++export interface SimpleAnalyticsEvent {+ added_iso: string;+ country_code: string;+ datapoint: string;+ document_referrer: string;+ hostname: string;+ lang_language: string;+ lang_region: string;+ path: string;+ query: string;+ screen_height: string;+ screen_width: string;+ session_id: string;+ user_agent: string;+ uuid: string;+}++export class SimpleAnalyticsImportMapper {+ private static readonly simpleAnalyticsEventSchema = z.object({+ added_iso: z.string().datetime(),+ country_code: z+ .string()+ .regex(/^[A-Z]{2}$/)+ .or(z.literal("")),+ datapoint: z.string().max(256),+ document_referrer: z.string().max(253 + 2048),+ hostname: z.string().max(253),+ lang_language: z.string().max(35),+ lang_region: z.string().max(35),+ path: z.string().max(2048),+ query: z+ .string()+ .max(2048)+ .transform(querystring => (querystring ? `?${querystring}` : "")),+ screen_height: z.string().regex(/^\d+$/),+ screen_width: z.string().regex(/^\d+$/),+ session_id: z.string().uuid(),+ user_agent: z.string().max(1024),+ uuid: z.string().uuid(),+ });++ static readonly simpleAnalyticsEventKeyOnlySchema = deriveKeyOnlySchema(+ SimpleAnalyticsImportMapper.simpleAnalyticsEventSchema+ );++ static transform(events: SimpleAnalyticsEvent[], site: number, importId: string): RybbitEvent[] {+ return events.reduce<RybbitEvent[]>((acc, event) => {+ const parsed = SimpleAnalyticsImportMapper.simpleAnalyticsEventSchema.safeParse(event);+ if (!parsed.success) {+ return acc;+ }++ const data = parsed.data;+ const ua = UAParser(data.user_agent);+ const screenWidth = parseInt(data.screen_width, 10);+ const screenHeight = parseInt(data.screen_height, 10);++ acc.push({+ site_id: site,+ timestamp: DateTime.fromISO(data.added_iso).toFormat("yyyy-MM-dd HH:mm:ss"),+ session_id: data.session_id,+ user_id: data.uuid,+ hostname: data.hostname,+ pathname: data.path,+ querystring: data.query,+ url_parameters: getAllUrlParams(data.query),+ page_title: "",+ referrer: clearSelfReferrer(data.document_referrer, data.hostname),+ channel: getChannel(data.document_referrer, data.query, data.hostname),+ browser: ua.browser.name || "",+ browser_version: ua.browser.major || "",+ operating_system: ua.os.name || "",+ operating_system_version: ua.os.version || "",+ language: data.lang_region ? `${data.lang_language}-${data.lang_region.toUpperCase()}` : data.lang_language,+ country: data.country_code,+ region: "",+ city: "",+ lat: 0,+ lon: 0,+ screen_width: screenWidth,+ screen_height: screenHeight,+ device_type: getDeviceType(screenWidth, screenHeight, ua),+ type: data.datapoint === "pageview" ? "pageview" : "custom_event",+ event_name: data.datapoint === "pageview" ? "" : data.datapoint,+ props: {},+ import_id: importId,+ });++ return acc;+ }, []);+ }+}