🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Add Plausible analytics import (#954)
Emilien Bidet1407c5d61575
diff --git a/client/src/components/SiteSettings/ImportManager.tsx b/client/src/components/SiteSettings/ImportManager.tsxindex 92f9cef10..269f536ca 100644--- a/client/src/components/SiteSettings/ImportManager.tsx+++ b/client/src/components/SiteSettings/ImportManager.tsx@@ -26,6 +26,7 @@ import { useGetSiteImports, useCreateSiteImport, useDeleteSiteImport } from "@/aimport { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";import { IS_CLOUD } from "@/lib/const";import { CsvParser } from "@/lib/import/csvParser";+import { PlausibleCsvParser } from "@/lib/import/plausibleParser";import { ImportPlatform } from "@/types/import";import { DisabledOverlay } from "@/components/DisabledOverlay";@@ -35,8 +36,6 @@ interface ImportManagerProps {}const CONFIRM_THRESHOLD = 100 * 1024 * 1024;-const ALLOWED_FILE_TYPES = ["text/csv"];-const ALLOWED_EXTENSIONS = [".csv"];function formatFileSize(bytes: number): string {const sizeInMB = bytes / 1024 / 1024;@@ -53,6 +52,7 @@ function formatPlatformName(platform: ImportPlatform): string {const platformNames: Record<ImportPlatform, string> = {umami: "Umami",simple_analytics: "Simple Analytics",+ plausible: "Plausible",};return platformNames[platform];}@@ -65,16 +65,22 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const [selectedPlatform, setSelectedPlatform] = useState<ImportPlatform | "">("");const [fileError, setFileError] = useState<string>("");const fileInputRef = useRef<HTMLInputElement>(null);- const workerManagerRef = useRef<CsvParser | null>(null);+ const workerManagerRef = useRef<CsvParser | PlausibleCsvParser | null>(null);- function validateFile(file: File | null): string {+ function validateFile(file: File | null, platform: ImportPlatform | ""): string {if (!file) {return t("Please select a file");}const extension = "." + file.name.split(".").pop()?.toLowerCase();- if (!ALLOWED_EXTENSIONS.includes(extension) && !ALLOWED_FILE_TYPES.includes(file.type)) {- return t("Only CSV files are accepted");+ if (platform === "plausible") {+ if (extension !== ".zip" && !["application/zip", "application/x-zip-compressed"].includes(file.type)) {+ return t("Please upload a ZIP file exported from Plausible");+ }+ } else {+ if (extension !== ".csv" && file.type !== "text/csv") {+ return t("Only CSV files are accepted");+ }}return "";@@ -93,7 +99,7 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {const file = e.target.files?.[0] || null;setSelectedFile(file);- setFileError(validateFile(file));+ setFileError(validateFile(file, selectedPlatform));};const onSubmit = (e: React.FormEvent) => {@@ -116,15 +122,28 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {onSuccess: response => {const { importId, allowedDateRange } = response.data;- workerManagerRef.current = new CsvParser(- siteId,- importId,- selectedPlatform,- allowedDateRange.earliestAllowedDate,- allowedDateRange.latestAllowedDate- );-- workerManagerRef.current.startImport(selectedFile);+ if (selectedPlatform === "plausible") {+ const parser = new PlausibleCsvParser(+ siteId,+ importId,+ allowedDateRange.earliestAllowedDate,+ allowedDateRange.latestAllowedDate+ );+ workerManagerRef.current = parser;+ parser.startImport(selectedFile).catch((err) => {+ console.error("Plausible import failed:", err);+ });+ } else {+ const parser = new CsvParser(+ siteId,+ importId,+ selectedPlatform,+ allowedDateRange.earliestAllowedDate,+ allowedDateRange.latestAllowedDate+ );+ workerManagerRef.current = parser;+ parser.startImport(selectedFile);+ }setSelectedFile(null);setSelectedPlatform("");@@ -207,7 +226,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<Alert><AlertCircle className="h-4 w-4" /><AlertDescription>- {t("You have an active import in progress. Please wait for it to complete before starting a new import.")}+ {t(+ "You have an active import in progress. Please wait for it to complete before starting a new import."+ )}</AlertDescription></Alert>)}@@ -216,13 +237,24 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {{/* Platform Selection */}<div className="space-y-2"><Label htmlFor="platform">{t("Platform")}</Label>- <Select value={selectedPlatform} onValueChange={(value: ImportPlatform) => setSelectedPlatform(value)}>+ <Select+ value={selectedPlatform}+ onValueChange={(value: ImportPlatform) => {+ setSelectedPlatform(value);+ setSelectedFile(null);+ setFileError("");+ if (fileInputRef.current) {+ fileInputRef.current.value = "";+ }+ }}+ ><SelectTrigger id="platform" disabled={disabled || createImportMutation.isPending || hasActiveImport}><SelectValue placeholder={t("Select platform")} /></SelectTrigger><SelectContent><SelectItem value="umami">Umami</SelectItem><SelectItem value="simple_analytics">Simple Analytics</SelectItem>+ <SelectItem value="plausible">Plausible</SelectItem></SelectContent></Select></div>@@ -231,13 +263,13 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<div className="space-y-2"><Label htmlFor="file" className="flex items-center gap-2"><FileText className="h-4 w-4" />- {t("CSV File")}+ {selectedPlatform === "plausible" ? t("ZIP File") : t("CSV File")}</Label><Inputref={fileInputRef}id="file"type="file"- accept=".csv"+ accept={selectedPlatform === "plausible" ? ".zip" : ".csv"}multiple={false}onChange={handleFileChange}disabled={disabled || createImportMutation.isPending || hasActiveImport}@@ -302,7 +334,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {) : error ? (<Alert variant="destructive"><AlertCircle className="h-4 w-4" />- <AlertDescription>{t("Failed to load import history. Please try refreshing the page.")}</AlertDescription>+ <AlertDescription>+ {t("Failed to load import history. Please try refreshing the page.")}+ </AlertDescription></Alert>) : !data?.data?.length ? (<div className="text-center py-8 text-muted-foreground">@@ -412,7 +446,10 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<AlertDialogHeader><AlertDialogTitle>{t("Confirm Large File Import")}</AlertDialogTitle><AlertDialogDescription>- {t("You're about to import a large file ({size}). This may take several minutes to process. Are you sure you want to continue?", { size: selectedFile ? formatFileSize(selectedFile.size) : "?" })}+ {t(+ "You're about to import a large file ({size}). This may take several minutes to process. Are you sure you want to continue?",+ { size: selectedFile ? formatFileSize(selectedFile.size) : "?" }+ )}</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter>@@ -428,7 +465,9 @@ export function ImportManager({ siteId, disabled }: ImportManagerProps) {<AlertDialogHeader><AlertDialogTitle>{t("Delete Import")}</AlertDialogTitle><AlertDialogDescription>- {t("Are you sure you want to delete this import? This action cannot be undone. The imported data will be permanently removed.")}+ {t(+ "Are you sure you want to delete this import? This action cannot be undone. The imported data will be permanently removed."+ )}</AlertDialogDescription></AlertDialogHeader><AlertDialogFooter>diff --git a/client/src/lib/import/plausibleParser.ts b/client/src/lib/import/plausibleParser.tsnew file mode 100644index 000000000..d2c094c77--- /dev/null+++ b/client/src/lib/import/plausibleParser.ts@@ -0,0 +1,762 @@+import JSZip from "jszip";+import Papa from "papaparse";+import { DateTime } from "luxon";+import { authedFetch } from "@/api/utils";++interface DistEntry<T> {+ value: T;+ weight: number;+}++interface BrowserInfo {+ browser: string;+ browser_version: string;+}++interface DeviceInfo {+ device_type: string;+}++interface OsInfo {+ operating_system: string;+ operating_system_version: string;+}++interface LocationInfo {+ country: string;+ region: string;+ city: string;+}++interface SourceInfo {+ referrer: string;+ utm_source: string;+ utm_medium: string;+ utm_campaign: string;+ utm_content: string;+ utm_term: string;+}++interface PlausibleSyntheticEvent {+ timestamp: string;+ session_id: string;+ user_id: string;+ hostname: string;+ pathname: string;+ querystring: string;+ referrer: string;+ browser: string;+ browser_version: string;+ operating_system: string;+ operating_system_version: string;+ device_type: string;+ country: string;+ region: string;+ city: string;+ type: string;+ event_name: string;+ props: string;+}++type DailyDist<T> = Map<string, DistEntry<T>[]>;++interface Session {+ sessionId: string;+ userId: string;+ hostname: string;+ browser: BrowserInfo;+ device: DeviceInfo;+ os: OsInfo;+ location: LocationInfo;+ source: SourceInfo;+ budget: number;+ pagesUsed: number;+ startSeconds: number;+ perPageSeconds: number;+}++interface DayPool {+ sessions: Session[];+ cursor: number;+}++interface PoolDistributions {+ browserDist: DailyDist<BrowserInfo>;+ deviceDist: DailyDist<DeviceInfo>;+ osDist: DailyDist<OsInfo>;+ locationDist: DailyDist<LocationInfo>;+ sourceDist: DailyDist<SourceInfo>;+ hostnameDist: DailyDist<string>;+ hostnameFallback: string;+}++const CHUNK_SIZE = 5000;+const SECONDS_IN_DAY = 86400;+const DEFAULT_PAGE_GAP_SECONDS = 30;++// Simple deterministic pseudo-random based on index+function deterministicPick<T>(+ items: DistEntry<T>[],+ index: number+): T {+ if (items.length === 0) {+ throw new Error("Cannot pick from empty distribution");+ }+ const totalWeight = items.reduce((sum, item) => sum + item.weight, 0);+ if (totalWeight === 0) return items[0].value;+ // Use a prime multiplier for better distribution+ const position = ((index * 7919) % totalWeight + totalWeight) % totalWeight;+ let cumulative = 0;+ for (const item of items) {+ cumulative += item.weight;+ if (position < cumulative) return item.value;+ }+ return items[items.length - 1].value;+}++function generateUUID(seed1: number, seed2: number): string {+ // Generate a deterministic UUID-like string from two seed numbers+ const hex = (n: number) => {+ return ((n * 2654435761) >>> 0).toString(16).padStart(8, "0");+ };+ const a = hex(seed1);+ const b = hex(seed2);+ const c = hex(seed1 + seed2);+ const d = hex(seed1 * 3 + seed2 * 7);+ return `${a}-${b.slice(0, 4)}-4${b.slice(5, 8)}-${c.slice(0, 4)}-${d}${c.slice(4, 8)}`.slice(+ 0,+ 36+ );+}++function identifyFile(+ headers: string[]+): string | null {+ const headerSet = new Set(headers);+ if (headerSet.has("browser") && headerSet.has("browser_version"))+ return "browsers";+ if (headerSet.has("device") && !headerSet.has("browser"))+ return "devices";+ if (headerSet.has("operating_system") && headerSet.has("operating_system_version"))+ return "operating_systems";+ if (headerSet.has("country") && headerSet.has("region") && headerSet.has("city"))+ return "locations";+ if (headerSet.has("source") && headerSet.has("utm_source"))+ return "sources";+ if (headerSet.has("page") && headerSet.has("hostname"))+ return "pages";+ if (headerSet.has("name") && headerSet.has("visitors") && headerSet.has("events"))+ return "custom_events";+ if (headerSet.has("entry_page")) return "entry_pages";+ if (headerSet.has("exit_page")) return "exit_pages";+ if (+ headerSet.has("visitors") &&+ headerSet.has("pageviews") &&+ headerSet.has("bounces") &&+ !headerSet.has("page") &&+ !headerSet.has("browser") &&+ !headerSet.has("device") &&+ !headerSet.has("country")+ )+ return "visitors";+ if (headerSet.has("property") && headerSet.has("value"))+ return "custom_props";+ return null;+}++function normalizeDevice(device: string): string {+ const lower = device.toLowerCase();+ if (lower === "desktop" || lower === "laptop") return "Desktop";+ if (lower === "mobile" || lower === "tablet") return "Mobile";+ return device;+}++const osMap: Record<string, string> = {+ "gnu/linux": "Linux",+ ubuntu: "Linux",+ mac: "macOS",+ "mac os": "macOS",+ "mac os x": "macOS",+ ios: "iOS",+ ipados: "iOS",+ android: "Android",+ windows: "Windows",+ "chrome os": "Chrome OS",+ chromeos: "Chrome OS",+};++function normalizeOs(os: string): string {+ if (!os) return "";+ return osMap[os.toLowerCase()] ?? os;+}++const browserMap: Record<string, string> = {+ "microsoft edge": "Edge",+ "samsung browser": "Samsung Internet",+ "yandex browser": "Yandex",+};++function normalizeBrowser(browser: string): string {+ if (!browser) return "";+ return browserMap[browser.toLowerCase()] ?? browser;+}++function buildReferrerUrl(source: string, referrer: string): string {+ const value = referrer || source;+ if (!value) return "";+ // Already has a scheme (e.g. "android-app://com.linkedin.android") — keep as-is.+ if (value.includes("://")) return value;+ // Friendly source names like "Brave" or "Bing" aren't URLs and can't become one.+ if (!value.includes(".") && !value.includes("/")) return "";+ return `https://${value}`;+}++function hashDate(date: string): number {+ let h = 0;+ for (let i = 0; i < date.length; i++) {+ h = ((h << 5) - h + date.charCodeAt(i)) | 0;+ }+ return Math.abs(h) || 1;+}++function clampSecondsToDay(seconds: number): number {+ if (seconds < 0) return 0;+ if (seconds > SECONDS_IN_DAY - 1) return SECONDS_IN_DAY - 1;+ return Math.floor(seconds);+}++function formatTimestamp(date: string, secondsOfDay: number): string {+ const s = clampSecondsToDay(secondsOfDay);+ const hours = Math.floor(s / 3600);+ const minutes = Math.floor((s % 3600) / 60);+ const seconds = s % 60;+ return `${date} ${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(seconds).padStart(2, "0")}`;+}++function buildQuerystring(utm: SourceInfo): string {+ const params = new URLSearchParams();+ if (utm.utm_source) params.set("utm_source", utm.utm_source);+ if (utm.utm_medium) params.set("utm_medium", utm.utm_medium);+ if (utm.utm_campaign) params.set("utm_campaign", utm.utm_campaign);+ if (utm.utm_content) params.set("utm_content", utm.utm_content);+ if (utm.utm_term) params.set("utm_term", utm.utm_term);+ const str = params.toString();+ return str ? `?${str}` : "";+}++export class PlausibleCsvParser {+ private cancelled = false;+ private readonly siteId: number;+ private readonly importId: string;+ private readonly earliestAllowedDate: DateTime;+ private readonly latestAllowedDate: DateTime;++ constructor(+ siteId: number,+ importId: string,+ earliestAllowedDate: string,+ latestAllowedDate: string+ ) {+ this.siteId = siteId;+ this.importId = importId;+ this.earliestAllowedDate = DateTime.fromFormat(+ earliestAllowedDate,+ "yyyy-MM-dd",+ { zone: "utc" }+ ).startOf("day");+ this.latestAllowedDate = DateTime.fromFormat(+ latestAllowedDate,+ "yyyy-MM-dd",+ { zone: "utc" }+ ).endOf("day");++ if (!this.earliestAllowedDate.isValid || !this.latestAllowedDate.isValid) {+ this.cancelled = true;+ }+ }++ cancel(): void {+ this.cancelled = true;+ }++ async startImport(file: File): Promise<void> {+ if (this.cancelled) return;++ try {+ // Phase 1: Extract CSVs from ZIP+ const zip = await JSZip.loadAsync(file);+ const csvFiles = new Map<string, Record<string, string>[]>();++ for (const [filename, zipEntry] of Object.entries(zip.files)) {+ if (this.cancelled) return;+ if (zipEntry.dir || !filename.endsWith(".csv")) continue;++ const csvText = await zipEntry.async("string");+ const parsed = Papa.parse<Record<string, string>>(csvText, {+ header: true,+ skipEmptyLines: "greedy",+ });++ if (parsed.data.length === 0 || !parsed.meta.fields) continue;++ const fileType = identifyFile(parsed.meta.fields);+ if (fileType) {+ csvFiles.set(fileType, parsed.data);+ }+ }++ const pagesData = csvFiles.get("pages");+ if (!pagesData || pagesData.length === 0) {+ // Must have pages data to generate events+ await this.uploadChunk([], true);+ return;+ }++ // Phase 2: Build daily distributions+ const browserDist = this.buildDist<BrowserInfo>(+ csvFiles.get("browsers"),+ (row) => ({+ browser: normalizeBrowser(row.browser || ""),+ browser_version: row.browser_version || "",+ }),+ (row) => parseInt(row.pageviews || "0", 10)+ );++ const deviceDist = this.buildDist<DeviceInfo>(+ csvFiles.get("devices"),+ (row) => ({ device_type: normalizeDevice(row.device || "") }),+ (row) => parseInt(row.pageviews || "0", 10)+ );++ const osDist = this.buildDist<OsInfo>(+ csvFiles.get("operating_systems"),+ (row) => ({+ operating_system: normalizeOs(row.operating_system || ""),+ operating_system_version: row.operating_system_version || "",+ }),+ (row) => parseInt(row.pageviews || "0", 10)+ );++ const locationDist = this.buildDist<LocationInfo>(+ csvFiles.get("locations"),+ (row) => ({+ country: row.country || "",+ region: row.region || "",+ // Plausible exports city as a numeric Geonames ID (e.g. "2654264"), not a name.+ // We have no lookup table, so drop it rather than store a fake city name.+ city: "",+ }),+ (row) => parseInt(row.pageviews || "0", 10)+ );++ const sourceDist = this.buildDist<SourceInfo>(+ csvFiles.get("sources"),+ (row) => ({+ referrer: buildReferrerUrl(row.source || "", row.referrer || ""),+ utm_source: row.utm_source || "",+ utm_medium: row.utm_medium || "",+ utm_campaign: row.utm_campaign || "",+ utm_content: row.utm_content || "",+ utm_term: row.utm_term || "",+ }),+ (row) => parseInt(row.pageviews || "0", 10)+ );++ // Custom-event rows have no hostname column, so derive one from the pages CSV.+ const hostnameDist = this.buildDist<string>(+ pagesData,+ (row) => row.hostname || "",+ (row) => parseInt(row.pageviews || "0", 10)+ );+ let hostnameFallback = "";+ let hostnameFallbackWeight = 0;+ for (const row of pagesData) {+ const h = row.hostname || "";+ if (!h) continue;+ const w = parseInt(row.pageviews || "0", 10);+ if (w > hostnameFallbackWeight) {+ hostnameFallbackWeight = w;+ hostnameFallback = h;+ }+ }++ // Phase 3: Build per-date session pools.+ //+ // Plausible exports daily totals (visitors, visits, bounces, pageviews) plus+ // per-page rows that re-aggregate the same activity. We use the daily totals+ // to size a pool of users + sessions for each date, give each session a+ // single set of context (browser/device/os/location/source/hostname) so the+ // context is consistent across all of that session's pageviews, and then+ // distribute pages from the pages CSV across the sessions. This produces+ // multi-page sessions with a real bounce rate, matching the source data.+ const visitorsData = csvFiles.get("visitors");+ const visitorsByDate = new Map<string, Record<string, string>>();+ if (visitorsData) {+ for (const row of visitorsData) {+ if (row.date) visitorsByDate.set(row.date, row);+ }+ }++ // Per-date pageview/visit totals from the pages CSV — used as a fallback+ // when no visitors row exists for a date.+ const pagesTotalsByDate = new Map<string, { pageviews: number; visits: number }>();+ for (const row of pagesData) {+ const date = row.date;+ if (!date) continue;+ const t = pagesTotalsByDate.get(date) ?? { pageviews: 0, visits: 0 };+ t.pageviews += parseInt(row.pageviews || "0", 10) || 0;+ t.visits += parseInt(row.visits || "0", 10) || 0;+ pagesTotalsByDate.set(date, t);+ }++ const dists: PoolDistributions = {+ browserDist,+ deviceDist,+ osDist,+ locationDist,+ sourceDist,+ hostnameDist,+ hostnameFallback,+ };++ const pools = new Map<string, DayPool>();+ for (const date of pagesTotalsByDate.keys()) {+ if (!this.isDateInRange(date)) continue;+ const pool = this.buildDayPool(+ date,+ visitorsByDate.get(date) ?? null,+ pagesTotalsByDate.get(date) ?? null,+ dists+ );+ if (pool.sessions.length > 0) pools.set(date, pool);+ }++ // Sort pages by entry-page weight (desc) within each date so the+ // highest-entry-likelihood pages are assigned first — they end up as+ // the first page of sessions, which gives Rybbit's "entry pages" report+ // a distribution that roughly matches Plausible's.+ const entryPagesData = csvFiles.get("entry_pages");+ const entryWeightByDatePage = new Map<string, number>();+ if (entryPagesData) {+ for (const row of entryPagesData) {+ if (!row.date || !row.entry_page) continue;+ const w = parseInt(row.entrances || "0", 10) || 0;+ entryWeightByDatePage.set(`${row.date}|${row.entry_page}`, w);+ }+ }+ const sortedPages = entryWeightByDatePage.size+ ? this.sortPagesByEntryWeight(pagesData, entryWeightByDatePage)+ : pagesData;++ // Phase 4: Generate pageview events using the pools.+ let buffer: PlausibleSyntheticEvent[] = [];++ for (const row of sortedPages) {+ if (this.cancelled) return;++ const date = row.date;+ if (!date) continue;+ const pool = pools.get(date);+ if (!pool) continue;++ const hostname = row.hostname || "";+ const page = row.page || "/";+ const pageviews = parseInt(row.pageviews || "0", 10);+ if (pageviews <= 0) continue;++ for (let i = 0; i < pageviews; i++) {+ const session = this.pickSessionForPageview(pool);+ const offsetSeconds =+ session.startSeconds + session.pagesUsed * session.perPageSeconds;+ const timestamp = formatTimestamp(date, offsetSeconds);++ buffer.push({+ timestamp,+ session_id: session.sessionId,+ user_id: session.userId,+ hostname,+ pathname: page,+ querystring: buildQuerystring(session.source),+ referrer: session.source.referrer,+ browser: session.browser.browser,+ browser_version: session.browser.browser_version,+ operating_system: session.os.operating_system,+ operating_system_version: session.os.operating_system_version,+ device_type: session.device.device_type,+ country: session.location.country,+ region: session.location.region,+ city: session.location.city,+ type: "pageview",+ event_name: "",+ props: "{}",+ });++ session.pagesUsed += 1;+ if (session.budget > 0) session.budget -= 1;++ if (buffer.length >= CHUNK_SIZE) {+ await this.uploadChunk(buffer, false);+ buffer = [];+ }+ }+ }++ // Phase 5: Generate custom events, attached to existing sessions so they+ // share user_ids with pageview activity for the same day.+ const customEventsData = csvFiles.get("custom_events");+ if (customEventsData) {+ for (const row of customEventsData) {+ if (this.cancelled) return;++ const date = row.date;+ if (!date) continue;+ const pool = pools.get(date);+ if (!pool || pool.sessions.length === 0) continue;++ const eventName = row.name || "";+ const path = row.path || "/";+ const linkUrl = row.link_url || "";+ const eventCount = parseInt(row.events || "0", 10);+ if (eventCount <= 0 || !eventName) continue;++ const propsStr = linkUrl ? JSON.stringify({ url: linkUrl }) : "{}";++ for (let i = 0; i < eventCount; i++) {+ const session = pool.sessions[i % pool.sessions.length];+ const offsetSeconds = Math.floor((i * SECONDS_IN_DAY) / eventCount);+ const timestamp = formatTimestamp(date, offsetSeconds);++ buffer.push({+ timestamp,+ session_id: session.sessionId,+ user_id: session.userId,+ hostname: session.hostname,+ pathname: path,+ querystring: buildQuerystring(session.source),+ referrer: session.source.referrer,+ browser: session.browser.browser,+ browser_version: session.browser.browser_version,+ operating_system: session.os.operating_system,+ operating_system_version: session.os.operating_system_version,+ device_type: session.device.device_type,+ country: session.location.country,+ region: session.location.region,+ city: session.location.city,+ type: "custom_event",+ event_name: eventName,+ props: propsStr,+ });++ if (buffer.length >= CHUNK_SIZE) {+ await this.uploadChunk(buffer, false);+ buffer = [];+ }+ }+ }+ }++ // Final flush+ if (this.cancelled) return;+ if (buffer.length > 0) {+ await this.uploadChunk(buffer, false);+ }+ await this.uploadChunk([], true);+ } catch (error) {+ console.error("Plausible import error:", error);+ // Try to mark import complete even on error+ try {+ await this.uploadChunk([], true);+ } catch {+ // ignore+ }+ }+ }++ private buildDist<T>(+ rows: Record<string, string>[] | undefined,+ extractor: (row: Record<string, string>) => T,+ weightFn: (row: Record<string, string>) => number+ ): DailyDist<T> {+ const dist: DailyDist<T> = new Map();+ if (!rows) return dist;++ for (const row of rows) {+ const date = row.date;+ if (!date) continue;+ const weight = weightFn(row);+ if (weight <= 0) continue;++ if (!dist.has(date)) {+ dist.set(date, []);+ }+ dist.get(date)!.push({ value: extractor(row), weight });+ }++ return dist;+ }++ private pickFromDist<T>(+ dist: DailyDist<T>,+ date: string,+ index: number,+ fallback: T+ ): T {+ const entries = dist.get(date);+ if (!entries || entries.length === 0) return fallback;+ return deterministicPick(entries, index);+ }++ private buildDayPool(+ date: string,+ visitorsRow: Record<string, string> | null,+ pagesTotals: { pageviews: number; visits: number } | null,+ dists: PoolDistributions+ ): DayPool {+ const visitorsRowPv = parseInt(visitorsRow?.pageviews ?? "", 10) || 0;+ const visitorsRowVisits = parseInt(visitorsRow?.visits ?? "", 10) || 0;+ const visitorsRowVisitors = parseInt(visitorsRow?.visitors ?? "", 10) || 0;+ const visitorsRowBounces = parseInt(visitorsRow?.bounces ?? "", 10) || 0;+ const visitorsRowDuration = parseInt(visitorsRow?.visit_duration ?? "", 10) || 0;++ const totalPageviews = Math.max(visitorsRowPv, pagesTotals?.pageviews ?? 0);+ const numSessions = Math.max(+ 1,+ visitorsRowVisits || pagesTotals?.visits || Math.max(1, Math.ceil(totalPageviews / 2))+ );+ const numUsers = Math.max(1, Math.min(numSessions, visitorsRowVisitors || numSessions));+ const numBounces = Math.min(numSessions, Math.max(0, visitorsRowBounces));++ if (totalPageviews <= 0) {+ return { sessions: [], cursor: 0 };+ }++ // Each session gets at least one pageview; bounces get exactly one.+ const minPageviews = Math.max(numSessions, totalPageviews);+ const nonBounce = numSessions - numBounces;+ const remainingForNonBounce = Math.max(0, minPageviews - numBounces);+ const baseNonBounce = nonBounce > 0 ? Math.floor(remainingForNonBounce / nonBounce) : 0;+ const extraNonBounce = nonBounce > 0 ? remainingForNonBounce - baseNonBounce * nonBounce : 0;++ const dateSeed = hashDate(date);+ const userIds: string[] = [];+ for (let i = 0; i < numUsers; i++) {+ userIds.push(generateUUID(dateSeed + i, dateSeed * 3 + i * 31337));+ }++ const sessions: Session[] = [];+ for (let i = 0; i < numSessions; i++) {+ const isBounce = i < numBounces;+ const budget = isBounce+ ? 1+ : Math.max(1, baseNonBounce + (i - numBounces < extraNonBounce ? 1 : 0));++ const startSeconds = Math.floor((i * SECONDS_IN_DAY) / numSessions);+ const avgDurationSeconds = visitorsRowDuration > 0 ? visitorsRowDuration / numSessions : 0;+ const perPageSeconds =+ budget > 1+ ? Math.max(1, Math.floor(avgDurationSeconds / (budget - 1)) || DEFAULT_PAGE_GAP_SECONDS)+ : DEFAULT_PAGE_GAP_SECONDS;++ sessions.push({+ sessionId: generateUUID(dateSeed * 7 + i, dateSeed * 11 + i * 7919),+ userId: userIds[i % numUsers],+ hostname: this.pickFromDist(dists.hostnameDist, date, i, dists.hostnameFallback),+ browser: this.pickFromDist(dists.browserDist, date, i, {+ browser: "",+ browser_version: "",+ }),+ device: this.pickFromDist(dists.deviceDist, date, i, { device_type: "" }),+ os: this.pickFromDist(dists.osDist, date, i, {+ operating_system: "",+ operating_system_version: "",+ }),+ location: this.pickFromDist(dists.locationDist, date, i, {+ country: "",+ region: "",+ city: "",+ }),+ source: this.pickFromDist(dists.sourceDist, date, i, {+ referrer: "",+ utm_source: "",+ utm_medium: "",+ utm_campaign: "",+ utm_content: "",+ utm_term: "",+ }),+ budget,+ pagesUsed: 0,+ startSeconds,+ perPageSeconds,+ });+ }++ return { sessions, cursor: 0 };+ }++ private pickSessionForPageview(pool: DayPool): Session {+ // Prefer sessions with remaining budget, advancing the cursor round-robin.+ for (let i = 0; i < pool.sessions.length; i++) {+ const idx = (pool.cursor + i) % pool.sessions.length;+ const session = pool.sessions[idx];+ if (session.budget > 0) {+ pool.cursor = (idx + 1) % pool.sessions.length;+ return session;+ }+ }+ // All budgets exhausted (page totals exceed the visitors total — possible+ // due to filtering). Over-allocate to keep events flowing rather than drop them.+ const session = pool.sessions[pool.cursor % pool.sessions.length];+ pool.cursor = (pool.cursor + 1) % pool.sessions.length;+ return session;+ }++ private sortPagesByEntryWeight(+ pagesData: Record<string, string>[],+ entryWeights: Map<string, number>+ ): Record<string, string>[] {+ // Group rows by date to preserve date locality, then sort within each+ // date by entry weight descending.+ const byDate = new Map<string, Record<string, string>[]>();+ for (const row of pagesData) {+ const date = row.date || "";+ if (!byDate.has(date)) byDate.set(date, []);+ byDate.get(date)!.push(row);+ }++ const out: Record<string, string>[] = [];+ for (const [date, rows] of byDate) {+ rows.sort((a, b) => {+ const aw = entryWeights.get(`${date}|${a.page}`) ?? 0;+ const bw = entryWeights.get(`${date}|${b.page}`) ?? 0;+ return bw - aw;+ });+ out.push(...rows);+ }+ return out;+ }++ private isDateInRange(dateStr: string): boolean {+ const date = DateTime.fromFormat(dateStr, "yyyy-MM-dd", { zone: "utc" });+ if (!date.isValid) return false;+ return date >= this.earliestAllowedDate && date <= this.latestAllowedDate;+ }++ private async uploadChunk(+ events: PlausibleSyntheticEvent[],+ isLastBatch: boolean+ ): Promise<void> {+ if (events.length === 0 && !isLastBatch) return;++ await authedFetch(+ `/sites/${this.siteId}/imports/${this.importId}/events`,+ undefined,+ {+ method: "POST",+ data: {+ events,+ isLastBatch,+ },+ }+ );+ }+}diff --git a/client/src/types/import.ts b/client/src/types/import.tsindex 8dde561c7..a3970d75b 100644--- a/client/src/types/import.ts+++ b/client/src/types/import.ts@@ -1 +1 @@-export type ImportPlatform = "umami" | "simple_analytics";+export type ImportPlatform = "umami" | "simple_analytics" | "plausible";diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 186b02f31..b6a2eec4c 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -2,8 +2,9 @@ import { FastifyReply, FastifyRequest } from "fastify";import { z } from "zod";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { updateImportProgress, completeImport, getImportById } from "../../services/import/importStatusManager.js";-import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappings/umami.js";-import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappings/simpleAnalytics.js";+import { UmamiEvent, UmamiImportMapper } from "../../services/import/mappers/umami.js";+import { SimpleAnalyticsEvent, SimpleAnalyticsImportMapper } from "../../services/import/mappers/simpleAnalytics.js";+import { PlausibleEvent, PlausibleImportMapper } from "../../services/import/mappers/plausible.js";import { importQuotaManager } from "../../services/import/importQuotaManager.js";import { db } from "../../db/postgres/postgres.js";import { organization, sites } from "../../db/postgres/schema.js";@@ -21,6 +22,7 @@ const batchImportRequestSchema = zevents: z.union([z.array(UmamiImportMapper.umamiEventKeyOnlySchema),z.array(SimpleAnalyticsImportMapper.simpleAnalyticsEventKeyOnlySchema),+ z.array(PlausibleImportMapper.plausibleEventKeyOnlySchema),]),isLastBatch: z.boolean().optional(),}),@@ -87,6 +89,8 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequetransformedEvents = UmamiImportMapper.transform(events as UmamiEvent[], siteId, importId);} else if (importRecord.platform === "simple_analytics") {transformedEvents = SimpleAnalyticsImportMapper.transform(events as SimpleAnalyticsEvent[], siteId, importId);+ } else if (importRecord.platform === "plausible") {+ transformedEvents = PlausibleImportMapper.transform(events as PlausibleEvent[], siteId, importId);} else {return reply.status(400).send({ error: "Unsupported platform" });}diff --git a/server/src/db/postgres/schema.ts b/server/src/db/postgres/schema.tsindex 9733bcdba..f5b447191 100644--- a/server/src/db/postgres/schema.ts+++ b/server/src/db/postgres/schema.ts@@ -663,7 +663,7 @@ export const cancellationFeedback = pgTable("cancellation_feedback", {createdAt: timestamp("created_at", { mode: "string" }).defaultNow().notNull(),});-export const importPlatforms = ["umami", "simple_analytics"] as const;+export const importPlatforms = ["umami", "simple_analytics", "plausible"] as const;export const importPlatformEnum = pgEnum("import_platform_enum", importPlatforms);diff --git a/server/src/services/import/mappers/plausible.test.ts b/server/src/services/import/mappers/plausible.test.tsnew file mode 100644index 000000000..67fa5ea48--- /dev/null+++ b/server/src/services/import/mappers/plausible.test.ts@@ -0,0 +1,176 @@+import { describe, it, expect } from "vitest";+import { PlausibleImportMapper } from "./plausible.js";++const TEST_SESSION_ID = "9e3779b1-3c6e-4362-da4f-81b88c81f013";+const TEST_USER_ID = "12345678-90ab-cdef-1234-567890abcdef";++function makeEvent(overrides: Record<string, string> = {}) {+ return {+ timestamp: "2024-06-15 14:30:00",+ session_id: TEST_SESSION_ID,+ user_id: TEST_USER_ID,+ hostname: "example.com",+ pathname: "/blog/post-1",+ querystring: "?utm_source=google",+ referrer: "https://google.com",+ browser: "Chrome",+ browser_version: "125",+ operating_system: "Windows",+ operating_system_version: "10",+ device_type: "Desktop",+ country: "US",+ region: "US-CA",+ city: "San Francisco",+ type: "pageview",+ event_name: "",+ props: "{}",+ ...overrides,+ };+}++describe("PlausibleImportMapper", () => {+ describe("transform", () => {+ it("should transform a valid pageview event", () => {+ const events = [makeEvent()];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");++ expect(result).toHaveLength(1);+ expect(result[0]).toMatchObject({+ site_id: 1,+ timestamp: "2024-06-15 14:30:00",+ session_id: TEST_SESSION_ID,+ user_id: TEST_USER_ID,+ hostname: "example.com",+ pathname: "/blog/post-1",+ querystring: "?utm_source=google",+ referrer: "https://google.com",+ browser: "Chrome",+ browser_version: "125",+ operating_system: "Windows",+ operating_system_version: "10",+ device_type: "Desktop",+ country: "US",+ region: "US-CA",+ city: "San Francisco",+ type: "pageview",+ event_name: "",+ props: {},+ import_id: "import-1",+ language: "",+ page_title: "",+ lat: 0,+ lon: 0,+ screen_width: 0,+ screen_height: 0,+ });+ });++ it("should transform a custom event", () => {+ const events = [+ makeEvent({+ type: "custom_event",+ event_name: "SignUp",+ props: '{"plan":"pro"}',+ }),+ ];+ const result = PlausibleImportMapper.transform(events, 2, "import-2");++ expect(result).toHaveLength(1);+ expect(result[0].type).toBe("custom_event");+ expect(result[0].event_name).toBe("SignUp");+ expect(result[0].props).toEqual({ plan: "pro" });+ });++ it("should skip events with invalid timestamps", () => {+ const events = [makeEvent({ timestamp: "not-a-date" })];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(0);+ });++ it("should skip events with invalid type", () => {+ const events = [makeEvent({ type: "unknown_type" })];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(0);+ });++ it("should handle empty props gracefully", () => {+ const events = [makeEvent({ props: "" })];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(1);+ expect(result[0].props).toEqual({});+ });++ it("should handle invalid JSON in props", () => {+ const events = [makeEvent({ props: "{bad json" })];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(1);+ expect(result[0].props).toEqual({});+ });++ it("should transform multiple events", () => {+ const events = [+ makeEvent({ pathname: "/page-1" }),+ makeEvent({ pathname: "/page-2" }),+ makeEvent({ pathname: "/page-3" }),+ ];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(3);+ expect(result.map((e) => e.pathname)).toEqual([+ "/page-1",+ "/page-2",+ "/page-3",+ ]);+ });++ it("should skip events with hostname exceeding max length", () => {+ const events = [makeEvent({ hostname: "a".repeat(254) })];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(0);+ });++ it("should handle empty optional fields", () => {+ const events = [+ makeEvent({+ referrer: "",+ browser_version: "",+ operating_system_version: "",+ country: "",+ region: "",+ city: "",+ querystring: "",+ }),+ ];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(1);+ expect(result[0].referrer).toBe("");+ expect(result[0].country).toBe("");+ });++ it("should compute channel from referrer", () => {+ const events = [+ makeEvent({ referrer: "https://google.com", querystring: "" }),+ ];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(1);+ // getChannel should classify google.com as organic search+ expect(result[0].channel).toBeTruthy();+ });++ it("should return empty array for empty input", () => {+ const result = PlausibleImportMapper.transform([], 1, "import-1");+ expect(result).toHaveLength(0);+ });++ it("should filter out invalid events and keep valid ones", () => {+ const events = [+ makeEvent({ pathname: "/valid" }),+ makeEvent({ timestamp: "bad" }),+ makeEvent({ pathname: "/also-valid" }),+ ];+ const result = PlausibleImportMapper.transform(events, 1, "import-1");+ expect(result).toHaveLength(2);+ expect(result[0].pathname).toBe("/valid");+ expect(result[1].pathname).toBe("/also-valid");+ });+ });+});diff --git a/server/src/services/import/mappers/plausible.ts b/server/src/services/import/mappers/plausible.tsnew file mode 100644index 000000000..ba9d73430--- /dev/null+++ b/server/src/services/import/mappers/plausible.ts@@ -0,0 +1,101 @@+import { clearSelfReferrer, getAllUrlParams } from "../../tracker/utils.js";+import { getChannel } from "../../tracker/getChannel.js";+import { RybbitEvent } from "./rybbit.js";+import { z } from "zod";+import { deriveKeyOnlySchema } from "./utils.js";++export type PlausibleEvent = z.input<+ typeof PlausibleImportMapper.plausibleEventKeyOnlySchema+>;++export class PlausibleImportMapper {+ private static readonly plausibleEventSchema = z.object({+ timestamp: z+ .string()+ .regex(+ /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/+ ),+ session_id: z.string().uuid(),+ user_id: z.string().uuid(),+ hostname: z.string().max(253),+ pathname: z.string().max(2048),+ querystring: z.string().max(2048),+ referrer: z.string().max(2048),+ browser: z.string().max(30),+ browser_version: z.string().max(20),+ operating_system: z.string().max(25),+ operating_system_version: z.string().max(20),+ device_type: z.string().max(20),+ country: z+ .string()+ .regex(/^[A-Z]{2}$/)+ .or(z.literal("")),+ region: z+ .string()+ .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)+ .or(z.literal("")),+ city: z.string().max(60),+ type: z.enum(["pageview", "custom_event"]),+ event_name: z.string().max(256),+ props: z.string().max(4096),+ });++ static readonly plausibleEventKeyOnlySchema = deriveKeyOnlySchema(+ PlausibleImportMapper.plausibleEventSchema+ );++ static transform(+ events: PlausibleEvent[],+ site: number,+ importId: string+ ): RybbitEvent[] {+ return events.reduce<RybbitEvent[]>((acc, event) => {+ const parsed =+ PlausibleImportMapper.plausibleEventSchema.safeParse(event);+ if (!parsed.success) return acc;++ const data = parsed.data;+ let parsedProps: Record<string, unknown> = {};+ try {+ parsedProps = JSON.parse(data.props || "{}");+ } catch {+ // ignore invalid JSON+ }++ const referrer = clearSelfReferrer(data.referrer, data.hostname.replace(/^www\./, ""));++ acc.push({+ site_id: site,+ timestamp: data.timestamp,+ session_id: data.session_id,+ user_id: data.user_id,+ hostname: data.hostname,+ pathname: data.pathname,+ querystring: data.querystring,+ url_parameters: getAllUrlParams(data.querystring),+ page_title: "",+ referrer,+ channel: getChannel(referrer, data.querystring, data.hostname),+ browser: data.browser,+ browser_version: data.browser_version,+ operating_system: data.operating_system,+ operating_system_version: data.operating_system_version,+ language: "",+ country: data.country,+ region: data.region,+ city: data.city,+ lat: 0,+ lon: 0,+ screen_width: 0,+ screen_height: 0,+ device_type: data.device_type,+ type: data.type,+ event_name: data.event_name,+ props: parsedProps,+ import_id: importId,+ });++ return acc;+ }, []);+ }+}diff --git a/server/src/services/import/mappings/rybbit.ts b/server/src/services/import/mappers/rybbit.tssimilarity index 100%rename from server/src/services/import/mappings/rybbit.tsrename to server/src/services/import/mappers/rybbit.tsdiff --git a/server/src/services/import/mappings/simpleAnalytics.ts b/server/src/services/import/mappers/simpleAnalytics.tssimilarity index 100%rename from server/src/services/import/mappings/simpleAnalytics.tsrename to server/src/services/import/mappers/simpleAnalytics.tsdiff --git a/server/src/services/import/mappings/umami.ts b/server/src/services/import/mappers/umami.tssimilarity index 100%rename from server/src/services/import/mappings/umami.tsrename to server/src/services/import/mappers/umami.tsdiff --git a/server/src/services/import/mappings/utils.ts b/server/src/services/import/mappers/utils.tssimilarity index 100%rename from server/src/services/import/mappings/utils.tsrename to server/src/services/import/mappers/utils.ts