🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
fix(import): keep Plausible imported session durations realistic (#1334)
Ryan19ae0783c513
diff --git a/client/src/lib/import/plausibleParser.test.ts b/client/src/lib/import/plausibleParser.test.tsindex 0c4764f4b..f43e8c295 100644--- a/client/src/lib/import/plausibleParser.test.ts+++ b/client/src/lib/import/plausibleParser.test.ts@@ -194,6 +194,26 @@ describe("pageview synthesis", () => {expect(events).toHaveLength(4);});+ it("gives the day's visit_duration to the non-bounced sessions, without truncating late ones", async () => {+ const { events } = await runImport({+ "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,3,5,3\n",+ "visitors.csv": "date,visitors,pageviews,bounces,visits,visit_duration\n2024-03-01,3,5,1,3,72000\n",+ });++ const bySession = new Map<string, string[]>();+ for (const e of events) bySession.set(e.session_id, [...(bySession.get(e.session_id) ?? []), e.timestamp]);+ const seconds = (ts: string) => {+ const [h, m, s] = ts.slice(11).split(":").map(Number);+ return h * 3600 + m * 60 + s;+ };+ const durations = [...bySession.values()].map(ts => seconds(ts.at(-1)!) - seconds(ts[0])).sort((a, b) => a - b);++ // 1 bounce (0s) and 2 visits of 10 hours each. The session that would start+ // at 16:00 starts at 13:59:59 instead, so it still ends before midnight.+ expect(durations).toEqual([0, 36000, 36000]);+ expect(durations.reduce((a, b) => a + b, 0)).toBe(72000);+ });+it("is deterministic — the same zip twice yields byte-identical events", async () => {const files = {"pages.csv": BASIC_PAGES,@@ -419,7 +439,7 @@ describe("referrers and utm parameters", () => {});describe("custom events", () => {- it("attaches custom events to the day's sessions, spread across the day", async () => {+ it("attaches custom events to the day's sessions at each session's last pageview", async () => {const { events } = await runImport({"pages.csv": BASIC_PAGES,"custom_events.csv": "date,name,visitors,events,path,link_url\n2024-03-01,Signup,2,2,/signup,\n",@@ -427,18 +447,34 @@ describe("custom events", () => {const custom = events.filter(e => e.type === "custom_event");expect(custom).toHaveLength(2);+ // Session 1 has pageviews at 00:00:00 and 00:00:30; session 2 has one at+ // 08:00:00. Neither event stretches its session.expect(custom.map(e => ({ timestamp: e.timestamp, event_name: e.event_name, pathname: e.pathname, props: e.props }))).toEqual([- { timestamp: "2024-03-01 00:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },- { timestamp: "2024-03-01 12:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },+ { timestamp: "2024-03-01 00:00:30", event_name: "Signup", pathname: "/signup", props: "{}" },+ { timestamp: "2024-03-01 08:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },]);+ expect(custom[0].session_id).toBe(events[0].session_id);+ expect(custom[1].session_id).toBe(events[1].session_id);// They reuse pageview sessions, so the day's user ids do not multiply.const pageviewSessions = new Set(events.filter(e => e.type === "pageview").map(e => e.session_id));expect(custom.every(e => pageviewSessions.has(e.session_id))).toBe(true);});+ it("skips Plausible's internal engagement events", async () => {+ const { events } = await runImport({+ "pages.csv": BASIC_PAGES,+ "custom_events.csv":+ "date,name,visitors,events,path,link_url\n" ++ "2024-03-01,engagement,3,40,,\n" ++ "2024-03-01,Signup,1,1,/signup,\n",+ });++ expect(events.filter(e => e.type === "custom_event").map(e => e.event_name)).toEqual(["Signup"]);+ });+it("carries link_url through as a url prop", async () => {const { events } = await runImport({"pages.csv": BASIC_PAGES,diff --git a/client/src/lib/import/plausibleParser.ts b/client/src/lib/import/plausibleParser.tsindex d2c094c77..a3931e142 100644--- a/client/src/lib/import/plausibleParser.ts+++ b/client/src/lib/import/plausibleParser.ts@@ -93,6 +93,9 @@ interface PoolDistributions {const CHUNK_SIZE = 5000;const SECONDS_IN_DAY = 86400;const DEFAULT_PAGE_GAP_SECONDS = 30;+// Plausible's own heartbeat for time-on-page and scroll depth. It is not a+// user-defined goal, and replaying it as custom events stretches sessions.+const PLAUSIBLE_INTERNAL_EVENTS = new Set(["engagement"]);// Simple deterministic pseudo-random based on indexfunction deterministicPick<T>(@@ -502,7 +505,9 @@ export class PlausibleCsvParser {}// Phase 5: Generate custom events, attached to existing sessions so they- // share user_ids with pageview activity for the same day.+ // share user_ids with pageview activity for the same day. Each event is+ // stamped at its session's last pageview so it never extends the session+ // beyond the duration Plausible reported.const customEventsData = csvFiles.get("custom_events");if (customEventsData) {for (const row of customEventsData) {@@ -518,12 +523,13 @@ export class PlausibleCsvParser {const linkUrl = row.link_url || "";const eventCount = parseInt(row.events || "0", 10);if (eventCount <= 0 || !eventName) continue;+ if (PLAUSIBLE_INTERNAL_EVENTS.has(eventName)) continue;const propsStr = linkUrl ? JSON.stringify({ url: linkUrl }) : "{}";for (let i = 0; i < eventCount; i++) {const session = pool.sessions[i % pool.sessions.length];- const offsetSeconds = Math.floor((i * SECONDS_IN_DAY) / eventCount);+ const offsetSeconds = session.startSeconds + Math.max(0, session.pagesUsed - 1) * session.perPageSeconds;const timestamp = formatTimestamp(date, offsetSeconds);buffer.push({@@ -643,6 +649,10 @@ export class PlausibleCsvParser {userIds.push(generateUUID(dateSeed + i, dateSeed * 3 + i * 31337));}+ // Single-pageview visits contribute nothing to Plausible's visit_duration,+ // so the day's total belongs to the non-bounced sessions alone.+ const avgDurationSeconds = visitorsRowDuration > 0 && nonBounce > 0 ? visitorsRowDuration / nonBounce : 0;+const sessions: Session[] = [];for (let i = 0; i < numSessions; i++) {const isBounce = i < numBounces;@@ -650,12 +660,17 @@ export class PlausibleCsvParser {? 1: Math.max(1, baseNonBounce + (i - numBounces < extraNonBounce ? 1 : 0));- const startSeconds = Math.floor((i * SECONDS_IN_DAY) / numSessions);- const avgDurationSeconds = visitorsRowDuration > 0 ? visitorsRowDuration / numSessions : 0;const perPageSeconds =- budget > 1- ? Math.max(1, Math.floor(avgDurationSeconds / (budget - 1)) || DEFAULT_PAGE_GAP_SECONDS)+ budget > 1 && avgDurationSeconds > 0+ ? Math.max(1, Math.round(avgDurationSeconds / (budget - 1))): DEFAULT_PAGE_GAP_SECONDS;+ // Start late sessions early enough to finish before midnight, so clamping+ // to the end of the day does not cut their duration short.+ const plannedSeconds = perPageSeconds * (budget - 1);+ const startSeconds = Math.min(+ Math.floor((i * SECONDS_IN_DAY) / numSessions),+ Math.max(0, SECONDS_IN_DAY - 1 - plannedSeconds)+ );sessions.push({sessionId: generateUUID(dateSeed * 7 + i, dateSeed * 11 + i * 7919),