Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Simple Analytics import (#741)

Edward Cui185e1f5abe85
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 282c445a8..1d4585b50 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -1,10 +1,11 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { authedFetch } from "@/api/utils";
import { APIResponse } from "@/api/types";
+import { ImportPlatform } from "@/types/import";
interface GetSiteImportsResponse {
importId: string;
- platform: "umami";
+ platform: ImportPlatform;
importedEvents: number;
skippedEvents: number;
invalidEvents: number;
@@ -37,7 +38,7 @@ export function useCreateSiteImport(site: number) {
const queryClient = useQueryClient();
return useMutation({
- mutationFn: async (data: { platform: "umami" }) => {
+ mutationFn: async (data: { platform: ImportPlatform }) => {
return await authedFetch<APIResponse<CreateSiteImportResponse>>(`/create-site-import/${site}`, undefined, {
method: "POST",
data,
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index bcfb36e9c..fdb87a337 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -25,6 +25,7 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/a
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { IS_CLOUD } from "@/lib/const";
import { CsvParser } from "@/lib/import/csvParser";
+import { ImportPlatform } from "@/types/import";
interface ImportManagerProps {
siteId: number;
@@ -59,11 +60,19 @@ function formatFileSize(bytes: number): string {
}
}
+function formatPlatformName(platform: ImportPlatform): string {
+ const platformNames: Record<ImportPlatform, string> = {
+ umami: "Umami",
+ simple_analytics: "Simple Analytics",
+ };
+ return platformNames[platform];
+}
+
export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const [showConfirmDialog, setShowConfirmDialog] = useState(false);
const [importToDelete, setImportToDelete] = useState<string | null>(null);
const [selectedFile, setSelectedFile] = useState<File | null>(null);
- const [selectedPlatform, setSelectedPlatform] = useState<"umami" | "">("");
+ const [selectedPlatform, setSelectedPlatform] = useState<ImportPlatform | "">("");
const [fileError, setFileError] = useState<string>("");
const fileInputRef = useRef<HTMLInputElement>(null);
const workerManagerRef = useRef<CsvParser | null>(null);
@@ -104,10 +113,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
onSuccess: response => {
const { importId, allowedDateRange } = response.data;
- workerManagerRef.current = new CsvParser();
-
- workerManagerRef.current.startImport(
- selectedFile,
+ workerManagerRef.current = new CsvParser(
siteId,
importId,
selectedPlatform,
@@ -115,6 +121,8 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
allowedDateRange.latestAllowedDate
);
+ workerManagerRef.current.startImport(selectedFile);
+
setSelectedFile(null);
setSelectedPlatform("");
setFileError("");
@@ -204,12 +212,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
{/* Platform Selection */}
<div className="space-y-2">
<Label htmlFor="platform">Platform</Label>
- <Select value={selectedPlatform} onValueChange={(value: "umami") => setSelectedPlatform(value)}>
+ <Select value={selectedPlatform} onValueChange={(value: ImportPlatform) => setSelectedPlatform(value)}>
<SelectTrigger id="platform" disabled={disabled || createImportMutation.isPending || hasActiveImport}>
<SelectValue placeholder="Select platform" />
</SelectTrigger>
<SelectContent>
<SelectItem value="umami">Umami</SelectItem>
+ <SelectItem value="simple_analytics">Simple Analytics</SelectItem>
</SelectContent>
</Select>
</div>
@@ -321,9 +330,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<TableRow key={imp.importId}>
<TableCell className="font-medium">{startedAt}</TableCell>
<TableCell>
- <Badge variant="outline" className="capitalize">
- {imp.platform}
- </Badge>
+ <Badge variant="outline">{formatPlatformName(imp.platform)}</Badge>
</TableCell>
<TableCell>
<Badge className={`${statusInfo.color} flex items-center gap-1`}>
diff --git a/client/src/lib/import/csvParser.ts b/client/src/lib/import/csvParser.ts
index 0b98d3bc8..839de4190 100644
--- a/client/src/lib/import/csvParser.ts
+++ b/client/src/lib/import/csvParser.ts
@@ -1,6 +1,7 @@
import Papa from "papaparse";
import { DateTime } from "luxon";
import { authedFetch } from "@/api/utils";
+import { ImportPlatform } from "@/types/import";
interface UmamiEvent {
session_id: string;
@@ -24,37 +25,52 @@ interface UmamiEvent {
created_at: string;
}
-export class CsvParser {
- private cancelled = false;
- private siteId: number = 0;
- private importId: string = "";
- private platform: "umami" = "umami";
-
- private earliestAllowedDate: DateTime | null = null;
- private latestAllowedDate: DateTime | null = null;
+interface SimpleAnalyticsEvent {
+ added_iso: string;
+ country_code: string;
+ datapoint: string;
+ document_referrer: string;
+ hostname: string;
+ lang_language: string;
+ lang_region: string;
+ path: string;
+ query: string;
+ screen_height: string;
+ screen_width: string;
+ session_id: string;
+ user_agent: string;
+ uuid: string;
+}
- startImport(
- file: File,
+export class CsvParser {
+ private cancelled: boolean = false;
+ private readonly siteId: number;
+ private readonly importId: string;
+ private readonly platform: ImportPlatform;
+ private readonly earliestAllowedDate: DateTime;
+ private readonly latestAllowedDate: DateTime;
+
+ constructor(
siteId: number,
importId: string,
- platform: "umami",
+ platform: ImportPlatform,
earliestAllowedDate: string,
latestAllowedDate: string
- ): void {
+ ) {
this.siteId = siteId;
this.importId = importId;
this.platform = platform;
-
this.earliestAllowedDate = DateTime.fromFormat(earliestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).startOf("day");
this.latestAllowedDate = DateTime.fromFormat(latestAllowedDate, "yyyy-MM-dd", { zone: "utc" }).endOf("day");
- if (!this.earliestAllowedDate.isValid) {
+ // Pre-validate dates during instantiation
+ if (!this.earliestAllowedDate.isValid || !this.latestAllowedDate.isValid) {
this.cancelled = true;
- return;
}
+ }
- if (!this.latestAllowedDate.isValid) {
- this.cancelled = true;
+ startImport(file: File): void {
+ if (this.cancelled) {
return;
}
@@ -70,16 +86,28 @@ export class CsvParser {
}
try {
- const validEvents: UmamiEvent[] = [];
- for (const row of results.data) {
- const event = this.transformRow(row);
- if (event && this.isDateInRange(event.created_at)) {
- validEvents.push(event);
+ if (this.platform === "umami") {
+ const validEvents: UmamiEvent[] = [];
+ for (const row of results.data) {
+ const event = this.transformRow(row);
+ if (event && this.isDateInRange((event as UmamiEvent).created_at)) {
+ validEvents.push(event as UmamiEvent);
+ }
+ }
+ if (validEvents.length > 0) {
+ await this.uploadChunk(validEvents, false);
+ }
+ } else {
+ const validEvents: SimpleAnalyticsEvent[] = [];
+ for (const row of results.data) {
+ const event = this.transformRow(row);
+ if (event && this.isDateInRange((event as SimpleAnalyticsEvent).added_iso)) {
+ validEvents.push(event as SimpleAnalyticsEvent);
+ }
+ }
+ if (validEvents.length > 0) {
+ await this.uploadChunk(validEvents, false);
}
- }
-
- if (validEvents.length > 0) {
- await this.uploadChunk(validEvents, false);
}
} catch (error) {
console.error("Error uploading chunk:", error);
@@ -105,55 +133,84 @@ export class CsvParser {
}
private isDateInRange(dateStr: string): boolean {
- const createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ // Handle both formats: "yyyy-MM-dd HH:mm:ss" (Umami) and ISO (Simple Analytics)
+ let createdAt = DateTime.fromFormat(dateStr, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
+ if (!createdAt.isValid) {
+ createdAt = DateTime.fromISO(dateStr, { zone: "utc" });
+ }
if (!createdAt.isValid) {
return false;
}
- if (this.earliestAllowedDate && createdAt < this.earliestAllowedDate) {
+ if (createdAt < this.earliestAllowedDate) {
return false;
}
- if (this.latestAllowedDate && createdAt > this.latestAllowedDate) {
+ if (createdAt > this.latestAllowedDate) {
return false;
}
return true;
}
- private transformRow(row: unknown): UmamiEvent | null {
+ private transformRow(row: unknown): UmamiEvent | SimpleAnalyticsEvent | null {
const rawEvent = row as Record<string, string>;
- const umamiEvent: UmamiEvent = {
- session_id: rawEvent.session_id,
- hostname: rawEvent.hostname,
- browser: rawEvent.browser,
- os: rawEvent.os,
- device: rawEvent.device,
- screen: rawEvent.screen,
- language: rawEvent.language,
- country: rawEvent.country,
- region: rawEvent.region,
- city: rawEvent.city,
- url_path: rawEvent.url_path,
- url_query: rawEvent.url_query,
- referrer_path: rawEvent.referrer_path,
- referrer_domain: rawEvent.referrer_domain,
- page_title: rawEvent.page_title,
- event_type: rawEvent.event_type,
- event_name: rawEvent.event_name,
- distinct_id: rawEvent.distinct_id,
- created_at: rawEvent.created_at,
- };
-
- if (!umamiEvent.created_at) {
- return null;
+ if (this.platform === "umami") {
+ const umamiEvent: UmamiEvent = {
+ session_id: rawEvent.session_id,
+ hostname: rawEvent.hostname,
+ browser: rawEvent.browser,
+ os: rawEvent.os,
+ device: rawEvent.device,
+ screen: rawEvent.screen,
+ language: rawEvent.language,
+ country: rawEvent.country,
+ region: rawEvent.region,
+ city: rawEvent.city,
+ url_path: rawEvent.url_path,
+ url_query: rawEvent.url_query,
+ referrer_path: rawEvent.referrer_path,
+ referrer_domain: rawEvent.referrer_domain,
+ page_title: rawEvent.page_title,
+ event_type: rawEvent.event_type,
+ event_name: rawEvent.event_name,
+ distinct_id: rawEvent.distinct_id,
+ created_at: rawEvent.created_at,
+ };
+
+ if (!umamiEvent.created_at) {
+ return null;
+ }
+
+ return umamiEvent;
+ } else {
+ const simpleAnalyticsEvent: SimpleAnalyticsEvent = {
+ added_iso: rawEvent.added_iso,
+ country_code: rawEvent.country_code,
+ datapoint: rawEvent.datapoint,
+ document_referrer: rawEvent.document_referrer,
+ hostname: rawEvent.hostname,
+ lang_language: rawEvent.lang_language,
+ lang_region: rawEvent.lang_region,
+ path: rawEvent.path,
+ query: rawEvent.query,
+ screen_height: rawEvent.screen_height,
+ screen_width: rawEvent.screen_width,
+ session_id: rawEvent.session_id,
+ user_agent: rawEvent.user_agent,
+ uuid: rawEvent.uuid,
+ };
+
+ if (!simpleAnalyticsEvent.added_iso) {
+ return null;
+ }
+
+ return simpleAnalyticsEvent;
}
-
- return umamiEvent;
}
- private async uploadChunk(events: UmamiEvent[], isLastBatch: boolean): Promise<void> {
+ private async uploadChunk(events: UmamiEvent[] | SimpleAnalyticsEvent[], isLastBatch: boolean): Promise<void> {
// Skip empty chunks unless it's the last one (needed for finalization)
if (events.length === 0 && !isLastBatch) {
return;
diff --git a/client/src/types/import.ts b/client/src/types/import.ts
new file mode 100644
index 000000000..8dde561c7
--- /dev/null
+++ b/client/src/types/import.ts
@@ -0,0 +1 @@
+export type ImportPlatform = "umami" | "simple_analytics";
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 583901247..e153f3558 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -3,7 +3,8 @@ import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";
-import { UmamiImportMapper } from "../../services/import/mappings/umami.js";
+import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappings/umami.js";
+import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappings/simpleAnalytics.js";
import { importQuotaManager } from "../../services/import/importQuotaManager.js";
import { db } from "../../db/postgres/postgres.js";
import { sites } from "../../db/postgres/schema.js";
@@ -16,7 +17,10 @@ const batchImportRequestSchema = z
importId: z.string().uuid(),
}),
body: z.object({
- events: z.array(UmamiImportMapper.umamiEventKeyOnlySchema),
+ events: z.union([
+ z.array(UmamiImportMapper.umamiEventKeyOnlySchema),
+ z.array(SimpleAnalyticsImportMapper.simpleAnalyticsEventKeyOnlySchema),
+ ]),
isLastBatch: z.boolean().optional(),
}),
})
@@ -68,7 +72,18 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
try {
const quotaTracker = await importQuotaManager.getTracker(siteRecord.organizationId);
- const transformedEvents = UmamiImportMapper.transform(events, site, importId);
+ let transformedEvents;
+ if (importRecord.platform === "umami") {
+ transformedEvents = UmamiImportMapper.transform(events as UmamiEvent[], site, importId);
+ } else if (importRecord.platform === "simple_analytics") {
+ transformedEvents = SimpleAnalyticsImportMapper.transform(
+ events as SimpleAnalyticsEvent[],
+ site,
+ importId
+ );
+ } else {
+ return reply.status(400).send({ error: "Unsupported platform" });
+ }
const invalidEventCount = events.length - transformedEvents.length;
const timestamps = transformedEvents.map(e => e.timestamp);
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 1cacb6fe2..ff417b03c 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -563,7 +563,7 @@ export const userAliases = pgTable(
]
);
-export const importPlatforms = ["umami"] as const;
+export const importPlatforms = ["umami", "simple_analytics"] as const;
export const importPlatformEnum = pgEnum("import_platform_enum", importPlatforms);
diff --git a/server/src/services/import/mappings/simpleAnalytics.ts b/server/src/services/import/mappings/simpleAnalytics.ts
new file mode 100644
index 000000000..1e29a06d2
--- /dev/null
+++ b/server/src/services/import/mappings/simpleAnalytics.ts
@@ -0,0 +1,101 @@
+import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";
+import { getChannel } from "../../tracker/getChannel.js";
+import { RybbitEvent } from "./rybbit.js";
+import { z } from "zod";
+import { UAParser } from "ua-parser-js";
+import { DateTime } from "luxon";
+import { getDeviceType } from "../../../utils.js";
+import { deriveKeyOnlySchema } from "./utils.js";
+
+export interface SimpleAnalyticsEvent {
+ added_iso: string;
+ country_code: string;
+ datapoint: string;
+ document_referrer: string;
+ hostname: string;
+ lang_language: string;
+ lang_region: string;
+ path: string;
+ query: string;
+ screen_height: string;
+ screen_width: string;
+ session_id: string;
+ user_agent: string;
+ uuid: string;
+}
+
+export class SimpleAnalyticsImportMapper {
+ private static readonly simpleAnalyticsEventSchema = z.object({
+ added_iso: z.string().datetime(),
+ country_code: z
+ .string()
+ .regex(/^[A-Z]{2}$/)
+ .or(z.literal("")),
+ datapoint: z.string().max(256),
+ document_referrer: z.string().max(253 + 2048),
+ hostname: z.string().max(253),
+ lang_language: z.string().max(35),
+ lang_region: z.string().max(35),
+ path: z.string().max(2048),
+ query: z
+ .string()
+ .max(2048)
+ .transform(querystring => (querystring ? `?${querystring}` : "")),
+ screen_height: z.string().regex(/^\d+$/),
+ screen_width: z.string().regex(/^\d+$/),
+ session_id: z.string().uuid(),
+ user_agent: z.string().max(1024),
+ uuid: z.string().uuid(),
+ });
+
+ static readonly simpleAnalyticsEventKeyOnlySchema = deriveKeyOnlySchema(
+ SimpleAnalyticsImportMapper.simpleAnalyticsEventSchema
+ );
+
+ static transform(events: SimpleAnalyticsEvent[], site: number, importId: string): RybbitEvent[] {
+ return events.reduce<RybbitEvent[]>((acc, event) => {
+ const parsed = SimpleAnalyticsImportMapper.simpleAnalyticsEventSchema.safeParse(event);
+ if (!parsed.success) {
+ return acc;
+ }
+
+ const data = parsed.data;
+ const ua = UAParser(data.user_agent);
+ const screenWidth = parseInt(data.screen_width, 10);
+ const screenHeight = parseInt(data.screen_height, 10);
+
+ acc.push({
+ site_id: site,
+ timestamp: DateTime.fromISO(data.added_iso).toFormat("yyyy-MM-dd HH:mm:ss"),
+ session_id: data.session_id,
+ user_id: data.uuid,
+ hostname: data.hostname,
+ pathname: data.path,
+ querystring: data.query,
+ url_parameters: getAllUrlParams(data.query),
+ page_title: "",
+ referrer: clearSelfReferrer(data.document_referrer, data.hostname),
+ channel: getChannel(data.document_referrer, data.query, data.hostname),
+ browser: ua.browser.name || "",
+ browser_version: ua.browser.major || "",
+ operating_system: ua.os.name || "",
+ operating_system_version: ua.os.version || "",
+ language: data.lang_region ? `${data.lang_language}-${data.lang_region.toUpperCase()}` : data.lang_language,
+ country: data.country_code,
+ region: "",
+ city: "",
+ lat: 0,
+ lon: 0,
+ screen_width: screenWidth,
+ screen_height: screenHeight,
+ device_type: getDeviceType(screenWidth, screenHeight, ua),
+ type: data.datapoint === "pageview" ? "pageview" : "custom_event",
+ event_name: data.datapoint === "pageview" ? "" : data.datapoint,
+ props: {},
+ import_id: importId,
+ });
+
+ return acc;
+ }, []);
+ }
+}