Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Add Plausible analytics import (#954)

Emilien Bidet1407c5d61575
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsx
index 92f9cef10..269f536ca 100644
--- a/client/src/components/SiteSettings/ImportManager.tsx
+++ b/client/src/components/SiteSettings/ImportManager.tsx
@@ -26,6 +26,7 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/a
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { IS_CLOUD } from "@/lib/const";
import { CsvParser } from "@/lib/import/csvParser";
+import { PlausibleCsvParser } from "@/lib/import/plausibleParser";
import { ImportPlatform } from "@/types/import";
import { DisabledOverlay } from "@/components/DisabledOverlay";
@@ -35,8 +36,6 @@ interface ImportManagerProps {
}
const CONFIRM_THRESHOLD = 100 * 1024 * 1024;
-const ALLOWED_FILE_TYPES = ["text/csv"];
-const ALLOWED_EXTENSIONS = [".csv"];
function formatFileSize(bytes: number): string {
const sizeInMB = bytes / 1024 / 1024;
@@ -53,6 +52,7 @@ function formatPlatformName(platform: ImportPlatform): string {
const platformNames: Record<ImportPlatform, string> = {
umami: "Umami",
simple_analytics: "Simple Analytics",
+ plausible: "Plausible",
};
return platformNames[platform];
}
@@ -65,16 +65,22 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const [selectedPlatform, setSelectedPlatform] = useState<ImportPlatform | "">("");
const [fileError, setFileError] = useState<string>("");
const fileInputRef = useRef<HTMLInputElement>(null);
- const workerManagerRef = useRef<CsvParser | null>(null);
+ const workerManagerRef = useRef<CsvParser | PlausibleCsvParser | null>(null);
- function validateFile(file: File | null): string {
+ function validateFile(file: File | null, platform: ImportPlatform | ""): string {
if (!file) {
return t("Please select a file");
}
const extension = "." + file.name.split(".").pop()?.toLowerCase();
- if (!ALLOWED_EXTENSIONS.includes(extension) && !ALLOWED_FILE_TYPES.includes(file.type)) {
- return t("Only CSV files are accepted");
+ if (platform === "plausible") {
+ if (extension !== ".zip" && !["application/zip", "application/x-zip-compressed"].includes(file.type)) {
+ return t("Please upload a ZIP file exported from Plausible");
+ }
+ } else {
+ if (extension !== ".csv" && file.type !== "text/csv") {
+ return t("Only CSV files are accepted");
+ }
}
return "";
@@ -93,7 +99,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0] || null;
setSelectedFile(file);
- setFileError(validateFile(file));
+ setFileError(validateFile(file, selectedPlatform));
};
const onSubmit = (e: React.FormEvent) => {
@@ -116,15 +122,28 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
onSuccess: response => {
const { importId, allowedDateRange } = response.data;
- workerManagerRef.current = new CsvParser(
- siteId,
- importId,
- selectedPlatform,
- allowedDateRange.earliestAllowedDate,
- allowedDateRange.latestAllowedDate
- );
-
- workerManagerRef.current.startImport(selectedFile);
+ if (selectedPlatform === "plausible") {
+ const parser = new PlausibleCsvParser(
+ siteId,
+ importId,
+ allowedDateRange.earliestAllowedDate,
+ allowedDateRange.latestAllowedDate
+ );
+ workerManagerRef.current = parser;
+ parser.startImport(selectedFile).catch((err) => {
+ console.error("Plausible import failed:", err);
+ });
+ } else {
+ const parser = new CsvParser(
+ siteId,
+ importId,
+ selectedPlatform,
+ allowedDateRange.earliestAllowedDate,
+ allowedDateRange.latestAllowedDate
+ );
+ workerManagerRef.current = parser;
+ parser.startImport(selectedFile);
+ }
setSelectedFile(null);
setSelectedPlatform("");
@@ -207,7 +226,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<Alert>
<AlertCircle className="h-4 w-4" />
<AlertDescription>
- {t("You have an active import in progress. Please wait for it to complete before starting a new import.")}
+ {t(
+ "You have an active import in progress. Please wait for it to complete before starting a new import."
+ )}
</AlertDescription>
</Alert>
)}
@@ -216,13 +237,24 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
{/* Platform Selection */}
<div className="space-y-2">
<Label htmlFor="platform">{t("Platform")}</Label>
- <Select value={selectedPlatform} onValueChange={(value: ImportPlatform) => setSelectedPlatform(value)}>
+ <Select
+ value={selectedPlatform}
+ onValueChange={(value: ImportPlatform) => {
+ setSelectedPlatform(value);
+ setSelectedFile(null);
+ setFileError("");
+ if (fileInputRef.current) {
+ fileInputRef.current.value = "";
+ }
+ }}
+ >
<SelectTrigger id="platform" disabled={disabled || createImportMutation.isPending || hasActiveImport}>
<SelectValue placeholder={t("Select platform")} />
</SelectTrigger>
<SelectContent>
<SelectItem value="umami">Umami</SelectItem>
<SelectItem value="simple_analytics">Simple Analytics</SelectItem>
+ <SelectItem value="plausible">Plausible</SelectItem>
</SelectContent>
</Select>
</div>
@@ -231,13 +263,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<div className="space-y-2">
<Label htmlFor="file" className="flex items-center gap-2">
<FileText className="h-4 w-4" />
- {t("CSV File")}
+ {selectedPlatform === "plausible" ? t("ZIP File") : t("CSV File")}
</Label>
<Input
ref={fileInputRef}
id="file"
type="file"
- accept=".csv"
+ accept={selectedPlatform === "plausible" ? ".zip" : ".csv"}
multiple={false}
onChange={handleFileChange}
disabled={disabled || createImportMutation.isPending || hasActiveImport}
@@ -302,7 +334,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
) : error ? (
<Alert variant="destructive">
<AlertCircle className="h-4 w-4" />
- <AlertDescription>{t("Failed to load import history. Please try refreshing the page.")}</AlertDescription>
+ <AlertDescription>
+ {t("Failed to load import history. Please try refreshing the page.")}
+ </AlertDescription>
</Alert>
) : !data?.data?.length ? (
<div className="text-center py-8 text-muted-foreground">
@@ -412,7 +446,10 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<AlertDialogHeader>
<AlertDialogTitle>{t("Confirm Large File Import")}</AlertDialogTitle>
<AlertDialogDescription>
- {t("You're about to import a large file ({size}). This may take several minutes to process. Are you sure you want to continue?", { size: selectedFile ? formatFileSize(selectedFile.size) : "?" })}
+ {t(
+ "You're about to import a large file ({size}). This may take several minutes to process. Are you sure you want to continue?",
+ { size: selectedFile ? formatFileSize(selectedFile.size) : "?" }
+ )}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
@@ -428,7 +465,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {
<AlertDialogHeader>
<AlertDialogTitle>{t("Delete Import")}</AlertDialogTitle>
<AlertDialogDescription>
- {t("Are you sure you want to delete this import? This action cannot be undone. The imported data will be permanently removed.")}
+ {t(
+ "Are you sure you want to delete this import? This action cannot be undone. The imported data will be permanently removed."
+ )}
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
diff --git a/client/src/lib/import/plausibleParser.ts b/client/src/lib/import/plausibleParser.ts
new file mode 100644
index 000000000..d2c094c77
--- /dev/null
+++ b/client/src/lib/import/plausibleParser.ts
@@ -0,0 +1,762 @@
+import JSZip from "jszip";
+import Papa from "papaparse";
+import { DateTime } from "luxon";
+import { authedFetch } from "@/api/utils";
+
+interface DistEntry<T> {
+ value: T;
+ weight: number;
+}
+
+interface BrowserInfo {
+ browser: string;
+ browser_version: string;
+}
+
+interface DeviceInfo {
+ device_type: string;
+}
+
+interface OsInfo {
+ operating_system: string;
+ operating_system_version: string;
+}
+
+interface LocationInfo {
+ country: string;
+ region: string;
+ city: string;
+}
+
+interface SourceInfo {
+ referrer: string;
+ utm_source: string;
+ utm_medium: string;
+ utm_campaign: string;
+ utm_content: string;
+ utm_term: string;
+}
+
+interface PlausibleSyntheticEvent {
+ timestamp: string;
+ session_id: string;
+ user_id: string;
+ hostname: string;
+ pathname: string;
+ querystring: string;
+ referrer: string;
+ browser: string;
+ browser_version: string;
+ operating_system: string;
+ operating_system_version: string;
+ device_type: string;
+ country: string;
+ region: string;
+ city: string;
+ type: string;
+ event_name: string;
+ props: string;
+}
+
+type DailyDist<T> = Map<string, DistEntry<T>[]>;
+
+interface Session {
+ sessionId: string;
+ userId: string;
+ hostname: string;
+ browser: BrowserInfo;
+ device: DeviceInfo;
+ os: OsInfo;
+ location: LocationInfo;
+ source: SourceInfo;
+ budget: number;
+ pagesUsed: number;
+ startSeconds: number;
+ perPageSeconds: number;
+}
+
+interface DayPool {
+ sessions: Session[];
+ cursor: number;
+}
+
+interface PoolDistributions {
+ browserDist: DailyDist<BrowserInfo>;
+ deviceDist: DailyDist<DeviceInfo>;
+ osDist: DailyDist<OsInfo>;
+ locationDist: DailyDist<LocationInfo>;
+ sourceDist: DailyDist<SourceInfo>;
+ hostnameDist: DailyDist<string>;
+ hostnameFallback: string;
+}
+
+const CHUNK_SIZE = 5000;
+const SECONDS_IN_DAY = 86400;
+const DEFAULT_PAGE_GAP_SECONDS = 30;
+
+// Simple deterministic pseudo-random based on index
+function deterministicPick<T>(
+ items: DistEntry<T>[],
+ index: number
+): T {
+ if (items.length === 0) {
+ throw new Error("Cannot pick from empty distribution");
+ }
+ const totalWeight = items.reduce((sum, item) => sum + item.weight, 0);
+ if (totalWeight === 0) return items[0].value;
+ // Use a prime multiplier for better distribution
+ const position = ((index * 7919) % totalWeight + totalWeight) % totalWeight;
+ let cumulative = 0;
+ for (const item of items) {
+ cumulative += item.weight;
+ if (position < cumulative) return item.value;
+ }
+ return items[items.length - 1].value;
+}
+
+function generateUUID(seed1: number, seed2: number): string {
+ // Generate a deterministic UUID-like string from two seed numbers
+ const hex = (n: number) => {
+ return ((n * 2654435761) >>> 0).toString(16).padStart(8, "0");
+ };
+ const a = hex(seed1);
+ const b = hex(seed2);
+ const c = hex(seed1 + seed2);
+ const d = hex(seed1 * 3 + seed2 * 7);
+ return `${a}-${b.slice(0, 4)}-4${b.slice(5, 8)}-${c.slice(0, 4)}-${d}${c.slice(4, 8)}`.slice(
+ 0,
+ 36
+ );
+}
+
+function identifyFile(
+ headers: string[]
+): string | null {
+ const headerSet = new Set(headers);
+ if (headerSet.has("browser") && headerSet.has("browser_version"))
+ return "browsers";
+ if (headerSet.has("device") && !headerSet.has("browser"))
+ return "devices";
+ if (headerSet.has("operating_system") && headerSet.has("operating_system_version"))
+ return "operating_systems";
+ if (headerSet.has("country") && headerSet.has("region") && headerSet.has("city"))
+ return "locations";
+ if (headerSet.has("source") && headerSet.has("utm_source"))
+ return "sources";
+ if (headerSet.has("page") && headerSet.has("hostname"))
+ return "pages";
+ if (headerSet.has("name") && headerSet.has("visitors") && headerSet.has("events"))
+ return "custom_events";
+ if (headerSet.has("entry_page")) return "entry_pages";
+ if (headerSet.has("exit_page")) return "exit_pages";
+ if (
+ headerSet.has("visitors") &&
+ headerSet.has("pageviews") &&
+ headerSet.has("bounces") &&
+ !headerSet.has("page") &&
+ !headerSet.has("browser") &&
+ !headerSet.has("device") &&
+ !headerSet.has("country")
+ )
+ return "visitors";
+ if (headerSet.has("property") && headerSet.has("value"))
+ return "custom_props";
+ return null;
+}
+
+function normalizeDevice(device: string): string {
+ const lower = device.toLowerCase();
+ if (lower === "desktop" || lower === "laptop") return "Desktop";
+ if (lower === "mobile" || lower === "tablet") return "Mobile";
+ return device;
+}
+
+const osMap: Record<string, string> = {
+ "gnu/linux": "Linux",
+ ubuntu: "Linux",
+ mac: "macOS",
+ "mac os": "macOS",
+ "mac os x": "macOS",
+ ios: "iOS",
+ ipados: "iOS",
+ android: "Android",
+ windows: "Windows",
+ "chrome os": "Chrome OS",
+ chromeos: "Chrome OS",
+};
+
+function normalizeOs(os: string): string {
+ if (!os) return "";
+ return osMap[os.toLowerCase()] ?? os;
+}
+
+const browserMap: Record<string, string> = {
+ "microsoft edge": "Edge",
+ "samsung browser": "Samsung Internet",
+ "yandex browser": "Yandex",
+};
+
+function normalizeBrowser(browser: string): string {
+ if (!browser) return "";
+ return browserMap[browser.toLowerCase()] ?? browser;
+}
+
+function buildReferrerUrl(source: string, referrer: string): string {
+ const value = referrer || source;
+ if (!value) return "";
+ // Already has a scheme (e.g. "android-app://com.linkedin.android") — keep as-is.
+ if (value.includes("://")) return value;
+ // Friendly source names like "Brave" or "Bing" aren't URLs and can't become one.
+ if (!value.includes(".") && !value.includes("/")) return "";
+ return `https://${value}`;
+}
+
+function hashDate(date: string): number {
+ let h = 0;
+ for (let i = 0; i < date.length; i++) {
+ h = ((h << 5) - h + date.charCodeAt(i)) | 0;
+ }
+ return Math.abs(h) || 1;
+}
+
+function clampSecondsToDay(seconds: number): number {
+ if (seconds < 0) return 0;
+ if (seconds > SECONDS_IN_DAY - 1) return SECONDS_IN_DAY - 1;
+ return Math.floor(seconds);
+}
+
+function formatTimestamp(date: string, secondsOfDay: number): string {
+ const s = clampSecondsToDay(secondsOfDay);
+ const hours = Math.floor(s / 3600);
+ const minutes = Math.floor((s % 3600) / 60);
+ const seconds = s % 60;
+ return `${date} ${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`;
+}
+
+function buildQuerystring(utm: SourceInfo): string {
+ const params = new URLSearchParams();
+ if (utm.utm_source) params.set("utm_source", utm.utm_source);
+ if (utm.utm_medium) params.set("utm_medium", utm.utm_medium);
+ if (utm.utm_campaign) params.set("utm_campaign", utm.utm_campaign);
+ if (utm.utm_content) params.set("utm_content", utm.utm_content);
+ if (utm.utm_term) params.set("utm_term", utm.utm_term);
+ const str = params.toString();
+ return str ? `?${str}` : "";
+}
+
+export class PlausibleCsvParser {
+ private cancelled = false;
+ private readonly siteId: number;
+ private readonly importId: string;
+ private readonly earliestAllowedDate: DateTime;
+ private readonly latestAllowedDate: DateTime;
+
+ constructor(
+ siteId: number,
+ importId: string,
+ earliestAllowedDate: string,
+ latestAllowedDate: string
+ ) {
+ this.siteId = siteId;
+ this.importId = importId;
+ this.earliestAllowedDate = DateTime.fromFormat(
+ earliestAllowedDate,
+ "yyyy-MM-dd",
+ { zone: "utc" }
+ ).startOf("day");
+ this.latestAllowedDate = DateTime.fromFormat(
+ latestAllowedDate,
+ "yyyy-MM-dd",
+ { zone: "utc" }
+ ).endOf("day");
+
+ if (!this.earliestAllowedDate.isValid || !this.latestAllowedDate.isValid) {
+ this.cancelled = true;
+ }
+ }
+
+ cancel(): void {
+ this.cancelled = true;
+ }
+
+ async startImport(file: File): Promise<void> {
+ if (this.cancelled) return;
+
+ try {
+ // Phase 1: Extract CSVs from ZIP
+ const zip = await JSZip.loadAsync(file);
+ const csvFiles = new Map<string, Record<string, string>[]>();
+
+ for (const [filename, zipEntry] of Object.entries(zip.files)) {
+ if (this.cancelled) return;
+ if (zipEntry.dir || !filename.endsWith(".csv")) continue;
+
+ const csvText = await zipEntry.async("string");
+ const parsed = Papa.parse<Record<string, string>>(csvText, {
+ header: true,
+ skipEmptyLines: "greedy",
+ });
+
+ if (parsed.data.length === 0 || !parsed.meta.fields) continue;
+
+ const fileType = identifyFile(parsed.meta.fields);
+ if (fileType) {
+ csvFiles.set(fileType, parsed.data);
+ }
+ }
+
+ const pagesData = csvFiles.get("pages");
+ if (!pagesData || pagesData.length === 0) {
+ // Must have pages data to generate events
+ await this.uploadChunk([], true);
+ return;
+ }
+
+ // Phase 2: Build daily distributions
+ const browserDist = this.buildDist<BrowserInfo>(
+ csvFiles.get("browsers"),
+ (row) => ({
+ browser: normalizeBrowser(row.browser || ""),
+ browser_version: row.browser_version || "",
+ }),
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+
+ const deviceDist = this.buildDist<DeviceInfo>(
+ csvFiles.get("devices"),
+ (row) => ({ device_type: normalizeDevice(row.device || "") }),
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+
+ const osDist = this.buildDist<OsInfo>(
+ csvFiles.get("operating_systems"),
+ (row) => ({
+ operating_system: normalizeOs(row.operating_system || ""),
+ operating_system_version: row.operating_system_version || "",
+ }),
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+
+ const locationDist = this.buildDist<LocationInfo>(
+ csvFiles.get("locations"),
+ (row) => ({
+ country: row.country || "",
+ region: row.region || "",
+ // Plausible exports city as a numeric Geonames ID (e.g. "2654264"), not a name.
+ // We have no lookup table, so drop it rather than store a fake city name.
+ city: "",
+ }),
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+
+ const sourceDist = this.buildDist<SourceInfo>(
+ csvFiles.get("sources"),
+ (row) => ({
+ referrer: buildReferrerUrl(row.source || "", row.referrer || ""),
+ utm_source: row.utm_source || "",
+ utm_medium: row.utm_medium || "",
+ utm_campaign: row.utm_campaign || "",
+ utm_content: row.utm_content || "",
+ utm_term: row.utm_term || "",
+ }),
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+
+ // Custom-event rows have no hostname column, so derive one from the pages CSV.
+ const hostnameDist = this.buildDist<string>(
+ pagesData,
+ (row) => row.hostname || "",
+ (row) => parseInt(row.pageviews || "0", 10)
+ );
+ let hostnameFallback = "";
+ let hostnameFallbackWeight = 0;
+ for (const row of pagesData) {
+ const h = row.hostname || "";
+ if (!h) continue;
+ const w = parseInt(row.pageviews || "0", 10);
+ if (w > hostnameFallbackWeight) {
+ hostnameFallbackWeight = w;
+ hostnameFallback = h;
+ }
+ }
+
+ // Phase 3: Build per-date session pools.
+ //
+ // Plausible exports daily totals (visitors, visits, bounces, pageviews) plus
+ // per-page rows that re-aggregate the same activity. We use the daily totals
+ // to size a pool of users + sessions for each date, give each session a
+ // single set of context (browser/device/os/location/source/hostname) so the
+ // context is consistent across all of that session's pageviews, and then
+ // distribute pages from the pages CSV across the sessions. This produces
+ // multi-page sessions with a real bounce rate, matching the source data.
+ const visitorsData = csvFiles.get("visitors");
+ const visitorsByDate = new Map<string, Record<string, string>>();
+ if (visitorsData) {
+ for (const row of visitorsData) {
+ if (row.date) visitorsByDate.set(row.date, row);
+ }
+ }
+
+ // Per-date pageview/visit totals from the pages CSV — used as a fallback
+ // when no visitors row exists for a date.
+ const pagesTotalsByDate = new Map<string, { pageviews: number; visits: number }>();
+ for (const row of pagesData) {
+ const date = row.date;
+ if (!date) continue;
+ const t = pagesTotalsByDate.get(date) ?? { pageviews: 0, visits: 0 };
+ t.pageviews += parseInt(row.pageviews || "0", 10) || 0;
+ t.visits += parseInt(row.visits || "0", 10) || 0;
+ pagesTotalsByDate.set(date, t);
+ }
+
+ const dists: PoolDistributions = {
+ browserDist,
+ deviceDist,
+ osDist,
+ locationDist,
+ sourceDist,
+ hostnameDist,
+ hostnameFallback,
+ };
+
+ const pools = new Map<string, DayPool>();
+ for (const date of pagesTotalsByDate.keys()) {
+ if (!this.isDateInRange(date)) continue;
+ const pool = this.buildDayPool(
+ date,
+ visitorsByDate.get(date) ?? null,
+ pagesTotalsByDate.get(date) ?? null,
+ dists
+ );
+ if (pool.sessions.length > 0) pools.set(date, pool);
+ }
+
+ // Sort pages by entry-page weight (desc) within each date so the
+ // highest-entry-likelihood pages are assigned first — they end up as
+ // the first page of sessions, which gives Rybbit's "entry pages" report
+ // a distribution that roughly matches Plausible's.
+ const entryPagesData = csvFiles.get("entry_pages");
+ const entryWeightByDatePage = new Map<string, number>();
+ if (entryPagesData) {
+ for (const row of entryPagesData) {
+ if (!row.date || !row.entry_page) continue;
+ const w = parseInt(row.entrances || "0", 10) || 0;
+ entryWeightByDatePage.set(`${row.date}|${row.entry_page}`, w);
+ }
+ }
+ const sortedPages = entryWeightByDatePage.size
+ ? this.sortPagesByEntryWeight(pagesData, entryWeightByDatePage)
+ : pagesData;
+
+ // Phase 4: Generate pageview events using the pools.
+ let buffer: PlausibleSyntheticEvent[] = [];
+
+ for (const row of sortedPages) {
+ if (this.cancelled) return;
+
+ const date = row.date;
+ if (!date) continue;
+ const pool = pools.get(date);
+ if (!pool) continue;
+
+ const hostname = row.hostname || "";
+ const page = row.page || "/";
+ const pageviews = parseInt(row.pageviews || "0", 10);
+ if (pageviews <= 0) continue;
+
+ for (let i = 0; i < pageviews; i++) {
+ const session = this.pickSessionForPageview(pool);
+ const offsetSeconds =
+ session.startSeconds + session.pagesUsed * session.perPageSeconds;
+ const timestamp = formatTimestamp(date, offsetSeconds);
+
+ buffer.push({
+ timestamp,
+ session_id: session.sessionId,
+ user_id: session.userId,
+ hostname,
+ pathname: page,
+ querystring: buildQuerystring(session.source),
+ referrer: session.source.referrer,
+ browser: session.browser.browser,
+ browser_version: session.browser.browser_version,
+ operating_system: session.os.operating_system,
+ operating_system_version: session.os.operating_system_version,
+ device_type: session.device.device_type,
+ country: session.location.country,
+ region: session.location.region,
+ city: session.location.city,
+ type: "pageview",
+ event_name: "",
+ props: "{}",
+ });
+
+ session.pagesUsed += 1;
+ if (session.budget > 0) session.budget -= 1;
+
+ if (buffer.length >= CHUNK_SIZE) {
+ await this.uploadChunk(buffer, false);
+ buffer = [];
+ }
+ }
+ }
+
+ // Phase 5: Generate custom events, attached to existing sessions so they
+ // share user_ids with pageview activity for the same day.
+ const customEventsData = csvFiles.get("custom_events");
+ if (customEventsData) {
+ for (const row of customEventsData) {
+ if (this.cancelled) return;
+
+ const date = row.date;
+ if (!date) continue;
+ const pool = pools.get(date);
+ if (!pool || pool.sessions.length === 0) continue;
+
+ const eventName = row.name || "";
+ const path = row.path || "/";
+ const linkUrl = row.link_url || "";
+ const eventCount = parseInt(row.events || "0", 10);
+ if (eventCount <= 0 || !eventName) continue;
+
+ const propsStr = linkUrl ? JSON.stringify({ url: linkUrl }) : "{}";
+
+ for (let i = 0; i < eventCount; i++) {
+ const session = pool.sessions[i % pool.sessions.length];
+ const offsetSeconds = Math.floor((i * SECONDS_IN_DAY) / eventCount);
+ const timestamp = formatTimestamp(date, offsetSeconds);
+
+ buffer.push({
+ timestamp,
+ session_id: session.sessionId,
+ user_id: session.userId,
+ hostname: session.hostname,
+ pathname: path,
+ querystring: buildQuerystring(session.source),
+ referrer: session.source.referrer,
+ browser: session.browser.browser,
+ browser_version: session.browser.browser_version,
+ operating_system: session.os.operating_system,
+ operating_system_version: session.os.operating_system_version,
+ device_type: session.device.device_type,
+ country: session.location.country,
+ region: session.location.region,
+ city: session.location.city,
+ type: "custom_event",
+ event_name: eventName,
+ props: propsStr,
+ });
+
+ if (buffer.length >= CHUNK_SIZE) {
+ await this.uploadChunk(buffer, false);
+ buffer = [];
+ }
+ }
+ }
+ }
+
+ // Final flush
+ if (this.cancelled) return;
+ if (buffer.length > 0) {
+ await this.uploadChunk(buffer, false);
+ }
+ await this.uploadChunk([], true);
+ } catch (error) {
+ console.error("Plausible import error:", error);
+ // Try to mark import complete even on error
+ try {
+ await this.uploadChunk([], true);
+ } catch {
+ // ignore
+ }
+ }
+ }
+
+ private buildDist<T>(
+ rows: Record<string, string>[] | undefined,
+ extractor: (row: Record<string, string>) => T,
+ weightFn: (row: Record<string, string>) => number
+ ): DailyDist<T> {
+ const dist: DailyDist<T> = new Map();
+ if (!rows) return dist;
+
+ for (const row of rows) {
+ const date = row.date;
+ if (!date) continue;
+ const weight = weightFn(row);
+ if (weight <= 0) continue;
+
+ if (!dist.has(date)) {
+ dist.set(date, []);
+ }
+ dist.get(date)!.push({ value: extractor(row), weight });
+ }
+
+ return dist;
+ }
+
+ private pickFromDist<T>(
+ dist: DailyDist<T>,
+ date: string,
+ index: number,
+ fallback: T
+ ): T {
+ const entries = dist.get(date);
+ if (!entries || entries.length === 0) return fallback;
+ return deterministicPick(entries, index);
+ }
+
+ private buildDayPool(
+ date: string,
+ visitorsRow: Record<string, string> | null,
+ pagesTotals: { pageviews: number; visits: number } | null,
+ dists: PoolDistributions
+ ): DayPool {
+ const visitorsRowPv = parseInt(visitorsRow?.pageviews ?? "", 10) || 0;
+ const visitorsRowVisits = parseInt(visitorsRow?.visits ?? "", 10) || 0;
+ const visitorsRowVisitors = parseInt(visitorsRow?.visitors ?? "", 10) || 0;
+ const visitorsRowBounces = parseInt(visitorsRow?.bounces ?? "", 10) || 0;
+ const visitorsRowDuration = parseInt(visitorsRow?.visit_duration ?? "", 10) || 0;
+
+ const totalPageviews = Math.max(visitorsRowPv, pagesTotals?.pageviews ?? 0);
+ const numSessions = Math.max(
+ 1,
+ visitorsRowVisits || pagesTotals?.visits || Math.max(1, Math.ceil(totalPageviews / 2))
+ );
+ const numUsers = Math.max(1, Math.min(numSessions, visitorsRowVisitors || numSessions));
+ const numBounces = Math.min(numSessions, Math.max(0, visitorsRowBounces));
+
+ if (totalPageviews <= 0) {
+ return { sessions: [], cursor: 0 };
+ }
+
+ // Each session gets at least one pageview; bounces get exactly one.
+ const minPageviews = Math.max(numSessions, totalPageviews);
+ const nonBounce = numSessions - numBounces;
+ const remainingForNonBounce = Math.max(0, minPageviews - numBounces);
+ const baseNonBounce = nonBounce > 0 ? Math.floor(remainingForNonBounce / nonBounce) : 0;
+ const extraNonBounce = nonBounce > 0 ? remainingForNonBounce - baseNonBounce * nonBounce : 0;
+
+ const dateSeed = hashDate(date);
+ const userIds: string[] = [];
+ for (let i = 0; i < numUsers; i++) {
+ userIds.push(generateUUID(dateSeed + i, dateSeed * 3 + i * 31337));
+ }
+
+ const sessions: Session[] = [];
+ for (let i = 0; i < numSessions; i++) {
+ const isBounce = i < numBounces;
+ const budget = isBounce
+ ? 1
+ : Math.max(1, baseNonBounce + (i - numBounces < extraNonBounce ? 1 : 0));
+
+ const startSeconds = Math.floor((i * SECONDS_IN_DAY) / numSessions);
+ const avgDurationSeconds = visitorsRowDuration > 0 ? visitorsRowDuration / numSessions : 0;
+ const perPageSeconds =
+ budget > 1
+ ? Math.max(1, Math.floor(avgDurationSeconds / (budget - 1)) || DEFAULT_PAGE_GAP_SECONDS)
+ : DEFAULT_PAGE_GAP_SECONDS;
+
+ sessions.push({
+ sessionId: generateUUID(dateSeed * 7 + i, dateSeed * 11 + i * 7919),
+ userId: userIds[i % numUsers],
+ hostname: this.pickFromDist(dists.hostnameDist, date, i, dists.hostnameFallback),
+ browser: this.pickFromDist(dists.browserDist, date, i, {
+ browser: "",
+ browser_version: "",
+ }),
+ device: this.pickFromDist(dists.deviceDist, date, i, { device_type: "" }),
+ os: this.pickFromDist(dists.osDist, date, i, {
+ operating_system: "",
+ operating_system_version: "",
+ }),
+ location: this.pickFromDist(dists.locationDist, date, i, {
+ country: "",
+ region: "",
+ city: "",
+ }),
+ source: this.pickFromDist(dists.sourceDist, date, i, {
+ referrer: "",
+ utm_source: "",
+ utm_medium: "",
+ utm_campaign: "",
+ utm_content: "",
+ utm_term: "",
+ }),
+ budget,
+ pagesUsed: 0,
+ startSeconds,
+ perPageSeconds,
+ });
+ }
+
+ return { sessions, cursor: 0 };
+ }
+
+ private pickSessionForPageview(pool: DayPool): Session {
+ // Prefer sessions with remaining budget, advancing the cursor round-robin.
+ for (let i = 0; i < pool.sessions.length; i++) {
+ const idx = (pool.cursor + i) % pool.sessions.length;
+ const session = pool.sessions[idx];
+ if (session.budget > 0) {
+ pool.cursor = (idx + 1) % pool.sessions.length;
+ return session;
+ }
+ }
+ // All budgets exhausted (page totals exceed the visitors total — possible
+ // due to filtering). Over-allocate to keep events flowing rather than drop them.
+ const session = pool.sessions[pool.cursor % pool.sessions.length];
+ pool.cursor = (pool.cursor + 1) % pool.sessions.length;
+ return session;
+ }
+
+ private sortPagesByEntryWeight(
+ pagesData: Record<string, string>[],
+ entryWeights: Map<string, number>
+ ): Record<string, string>[] {
+ // Group rows by date to preserve date locality, then sort within each
+ // date by entry weight descending.
+ const byDate = new Map<string, Record<string, string>[]>();
+ for (const row of pagesData) {
+ const date = row.date || "";
+ if (!byDate.has(date)) byDate.set(date, []);
+ byDate.get(date)!.push(row);
+ }
+
+ const out: Record<string, string>[] = [];
+ for (const [date, rows] of byDate) {
+ rows.sort((a, b) => {
+ const aw = entryWeights.get(`${date}|${a.page}`) ?? 0;
+ const bw = entryWeights.get(`${date}|${b.page}`) ?? 0;
+ return bw - aw;
+ });
+ out.push(...rows);
+ }
+ return out;
+ }
+
+ private isDateInRange(dateStr: string): boolean {
+ const date = DateTime.fromFormat(dateStr, "yyyy-MM-dd", { zone: "utc" });
+ if (!date.isValid) return false;
+ return date >= this.earliestAllowedDate && date <= this.latestAllowedDate;
+ }
+
+ private async uploadChunk(
+ events: PlausibleSyntheticEvent[],
+ isLastBatch: boolean
+ ): Promise<void> {
+ if (events.length === 0 && !isLastBatch) return;
+
+ await authedFetch(
+ `/sites/${this.siteId}/imports/${this.importId}/events`,
+ undefined,
+ {
+ method: "POST",
+ data: {
+ events,
+ isLastBatch,
+ },
+ }
+ );
+ }
+}
diff --git a/client/src/types/import.ts b/client/src/types/import.ts
index 8dde561c7..a3970d75b 100644
--- a/client/src/types/import.ts
+++ b/client/src/types/import.ts
@@ -1 +1 @@
-export type ImportPlatform = "umami" | "simple_analytics";
+export type ImportPlatform = "umami" | "simple_analytics" | "plausible";
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 186b02f31..b6a2eec4c 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -2,8 +2,9 @@ import { FastifyReply, FastifyRequest } from "fastify";
import { z } from "zod";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";
-import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappings/umami.js";
-import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappings/simpleAnalytics.js";
+import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappers/umami.js";
+import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappers/simpleAnalytics.js";
+import { PlausibleEvent, PlausibleImportMapper } from "../../services/import/mappers/plausible.js";
import { importQuotaManager } from "../../services/import/importQuotaManager.js";
import { db } from "../../db/postgres/postgres.js";
import { organization, sites } from "../../db/postgres/schema.js";
@@ -21,6 +22,7 @@ const batchImportRequestSchema = z
events: z.union([
z.array(UmamiImportMapper.umamiEventKeyOnlySchema),
z.array(SimpleAnalyticsImportMapper.simpleAnalyticsEventKeyOnlySchema),
+ z.array(PlausibleImportMapper.plausibleEventKeyOnlySchema),
]),
isLastBatch: z.boolean().optional(),
}),
@@ -87,6 +89,8 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
transformedEvents = UmamiImportMapper.transform(events as UmamiEvent[], siteId, importId);
} else if (importRecord.platform === "simple_analytics") {
transformedEvents = SimpleAnalyticsImportMapper.transform(events as SimpleAnalyticsEvent[], siteId, importId);
+ } else if (importRecord.platform === "plausible") {
+ transformedEvents = PlausibleImportMapper.transform(events as PlausibleEvent[], siteId, importId);
} else {
return reply.status(400).send({ error: "Unsupported platform" });
}
diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.ts
index 9733bcdba..f5b447191 100644
--- a/server/src/db/postgres/schema.ts
+++ b/server/src/db/postgres/schema.ts
@@ -663,7 +663,7 @@ export const cancellationFeedback = pgTable("cancellation_feedback", {
createdAt: timestamp("created_at", { mode: "string" }).defaultNow().notNull(),
});
-export const importPlatforms = ["umami", "simple_analytics"] as const;
+export const importPlatforms = ["umami", "simple_analytics", "plausible"] as const;
export const importPlatformEnum = pgEnum("import_platform_enum", importPlatforms);
diff --git a/server/src/services/import/mappers/plausible.test.ts b/server/src/services/import/mappers/plausible.test.ts
new file mode 100644
index 000000000..67fa5ea48
--- /dev/null
+++ b/server/src/services/import/mappers/plausible.test.ts
@@ -0,0 +1,176 @@
+import { describe, it, expect } from "vitest";
+import { PlausibleImportMapper } from "./plausible.js";
+
+const TEST_SESSION_ID = "9e3779b1-3c6e-4362-da4f-81b88c81f013";
+const TEST_USER_ID = "12345678-90ab-cdef-1234-567890abcdef";
+
+function makeEvent(overrides: Record<string, string> = {}) {
+ return {
+ timestamp: "2024-06-15 14:30:00",
+ session_id: TEST_SESSION_ID,
+ user_id: TEST_USER_ID,
+ hostname: "example.com",
+ pathname: "/blog/post-1",
+ querystring: "?utm_source=google",
+ referrer: "https://google.com",
+ browser: "Chrome",
+ browser_version: "125",
+ operating_system: "Windows",
+ operating_system_version: "10",
+ device_type: "Desktop",
+ country: "US",
+ region: "US-CA",
+ city: "San Francisco",
+ type: "pageview",
+ event_name: "",
+ props: "{}",
+ ...overrides,
+ };
+}
+
+describe("PlausibleImportMapper", () => {
+ describe("transform", () => {
+ it("should transform a valid pageview event", () => {
+ const events = [makeEvent()];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+
+ expect(result).toHaveLength(1);
+ expect(result[0]).toMatchObject({
+ site_id: 1,
+ timestamp: "2024-06-15 14:30:00",
+ session_id: TEST_SESSION_ID,
+ user_id: TEST_USER_ID,
+ hostname: "example.com",
+ pathname: "/blog/post-1",
+ querystring: "?utm_source=google",
+ referrer: "https://google.com",
+ browser: "Chrome",
+ browser_version: "125",
+ operating_system: "Windows",
+ operating_system_version: "10",
+ device_type: "Desktop",
+ country: "US",
+ region: "US-CA",
+ city: "San Francisco",
+ type: "pageview",
+ event_name: "",
+ props: {},
+ import_id: "import-1",
+ language: "",
+ page_title: "",
+ lat: 0,
+ lon: 0,
+ screen_width: 0,
+ screen_height: 0,
+ });
+ });
+
+ it("should transform a custom event", () => {
+ const events = [
+ makeEvent({
+ type: "custom_event",
+ event_name: "SignUp",
+ props: '{"plan":"pro"}',
+ }),
+ ];
+ const result = PlausibleImportMapper.transform(events, 2, "import-2");
+
+ expect(result).toHaveLength(1);
+ expect(result[0].type).toBe("custom_event");
+ expect(result[0].event_name).toBe("SignUp");
+ expect(result[0].props).toEqual({ plan: "pro" });
+ });
+
+ it("should skip events with invalid timestamps", () => {
+ const events = [makeEvent({ timestamp: "not-a-date" })];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(0);
+ });
+
+ it("should skip events with invalid type", () => {
+ const events = [makeEvent({ type: "unknown_type" })];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(0);
+ });
+
+ it("should handle empty props gracefully", () => {
+ const events = [makeEvent({ props: "" })];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(1);
+ expect(result[0].props).toEqual({});
+ });
+
+ it("should handle invalid JSON in props", () => {
+ const events = [makeEvent({ props: "{bad json" })];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(1);
+ expect(result[0].props).toEqual({});
+ });
+
+ it("should transform multiple events", () => {
+ const events = [
+ makeEvent({ pathname: "/page-1" }),
+ makeEvent({ pathname: "/page-2" }),
+ makeEvent({ pathname: "/page-3" }),
+ ];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(3);
+ expect(result.map((e) => e.pathname)).toEqual([
+ "/page-1",
+ "/page-2",
+ "/page-3",
+ ]);
+ });
+
+ it("should skip events with hostname exceeding max length", () => {
+ const events = [makeEvent({ hostname: "a".repeat(254) })];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(0);
+ });
+
+ it("should handle empty optional fields", () => {
+ const events = [
+ makeEvent({
+ referrer: "",
+ browser_version: "",
+ operating_system_version: "",
+ country: "",
+ region: "",
+ city: "",
+ querystring: "",
+ }),
+ ];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(1);
+ expect(result[0].referrer).toBe("");
+ expect(result[0].country).toBe("");
+ });
+
+ it("should compute channel from referrer", () => {
+ const events = [
+ makeEvent({ referrer: "https://google.com", querystring: "" }),
+ ];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(1);
+ // getChannel should classify google.com as organic search
+ expect(result[0].channel).toBeTruthy();
+ });
+
+ it("should return empty array for empty input", () => {
+ const result = PlausibleImportMapper.transform([], 1, "import-1");
+ expect(result).toHaveLength(0);
+ });
+
+ it("should filter out invalid events and keep valid ones", () => {
+ const events = [
+ makeEvent({ pathname: "/valid" }),
+ makeEvent({ timestamp: "bad" }),
+ makeEvent({ pathname: "/also-valid" }),
+ ];
+ const result = PlausibleImportMapper.transform(events, 1, "import-1");
+ expect(result).toHaveLength(2);
+ expect(result[0].pathname).toBe("/valid");
+ expect(result[1].pathname).toBe("/also-valid");
+ });
+ });
+});
diff --git a/server/src/services/import/mappers/plausible.ts b/server/src/services/import/mappers/plausible.ts
new file mode 100644
index 000000000..ba9d73430
--- /dev/null
+++ b/server/src/services/import/mappers/plausible.ts
@@ -0,0 +1,101 @@
+import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";
+import { getChannel } from "../../tracker/getChannel.js";
+import { RybbitEvent } from "./rybbit.js";
+import { z } from "zod";
+import { deriveKeyOnlySchema } from "./utils.js";
+
+export type PlausibleEvent = z.input<
+ typeof PlausibleImportMapper.plausibleEventKeyOnlySchema
+>;
+
+export class PlausibleImportMapper {
+ private static readonly plausibleEventSchema = z.object({
+ timestamp: z
+ .string()
+ .regex(
+ /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/
+ ),
+ session_id: z.string().uuid(),
+ user_id: z.string().uuid(),
+ hostname: z.string().max(253),
+ pathname: z.string().max(2048),
+ querystring: z.string().max(2048),
+ referrer: z.string().max(2048),
+ browser: z.string().max(30),
+ browser_version: z.string().max(20),
+ operating_system: z.string().max(25),
+ operating_system_version: z.string().max(20),
+ device_type: z.string().max(20),
+ country: z
+ .string()
+ .regex(/^[A-Z]{2}$/)
+ .or(z.literal("")),
+ region: z
+ .string()
+ .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)
+ .or(z.literal("")),
+ city: z.string().max(60),
+ type: z.enum(["pageview", "custom_event"]),
+ event_name: z.string().max(256),
+ props: z.string().max(4096),
+ });
+
+ static readonly plausibleEventKeyOnlySchema = deriveKeyOnlySchema(
+ PlausibleImportMapper.plausibleEventSchema
+ );
+
+ static transform(
+ events: PlausibleEvent[],
+ site: number,
+ importId: string
+ ): RybbitEvent[] {
+ return events.reduce<RybbitEvent[]>((acc, event) => {
+ const parsed =
+ PlausibleImportMapper.plausibleEventSchema.safeParse(event);
+ if (!parsed.success) return acc;
+
+ const data = parsed.data;
+ let parsedProps: Record<string, unknown> = {};
+ try {
+ parsedProps = JSON.parse(data.props || "{}");
+ } catch {
+ // ignore invalid JSON
+ }
+
+ const referrer = clearSelfReferrer(data.referrer, data.hostname.replace(/^www\./, ""));
+
+ acc.push({
+ site_id: site,
+ timestamp: data.timestamp,
+ session_id: data.session_id,
+ user_id: data.user_id,
+ hostname: data.hostname,
+ pathname: data.pathname,
+ querystring: data.querystring,
+ url_parameters: getAllUrlParams(data.querystring),
+ page_title: "",
+ referrer,
+ channel: getChannel(referrer, data.querystring, data.hostname),
+ browser: data.browser,
+ browser_version: data.browser_version,
+ operating_system: data.operating_system,
+ operating_system_version: data.operating_system_version,
+ language: "",
+ country: data.country,
+ region: data.region,
+ city: data.city,
+ lat: 0,
+ lon: 0,
+ screen_width: 0,
+ screen_height: 0,
+ device_type: data.device_type,
+ type: data.type,
+ event_name: data.event_name,
+ props: parsedProps,
+ import_id: importId,
+ });
+
+ return acc;
+ }, []);
+ }
+}
diff --git a/server/src/services/import/mappings/rybbit.ts b/server/src/services/import/mappers/rybbit.ts
similarity index 100%
rename from server/src/services/import/mappings/rybbit.ts
rename to server/src/services/import/mappers/rybbit.ts
diff --git a/server/src/services/import/mappings/simpleAnalytics.ts b/server/src/services/import/mappers/simpleAnalytics.ts
similarity index 100%
rename from server/src/services/import/mappings/simpleAnalytics.ts
rename to server/src/services/import/mappers/simpleAnalytics.ts
diff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappers/umami.ts
similarity index 100%
rename from server/src/services/import/mappings/umami.ts
rename to server/src/services/import/mappers/umami.ts
diff --git a/server/src/services/import/mappings/utils.ts b/server/src/services/import/mappers/utils.ts
similarity index 100%
rename from server/src/services/import/mappings/utils.ts
rename to server/src/services/import/mappers/utils.ts