Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

fix(import): keep Plausible imported session durations realistic (#1334)

Ryan19ae0783c513
diff --git a/client/src/lib/import/plausibleParser.test.ts b/client/src/lib/import/plausibleParser.test.ts
index 0c4764f4b..f43e8c295 100644
--- a/client/src/lib/import/plausibleParser.test.ts
+++ b/client/src/lib/import/plausibleParser.test.ts
@@ -194,6 +194,26 @@ describe("pageview synthesis", () => {
expect(events).toHaveLength(4);
});
+ it("gives the day's visit_duration to the non-bounced sessions, without truncating late ones", async () => {
+ const { events } = await runImport({
+ "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,3,5,3\n",
+ "visitors.csv": "date,visitors,pageviews,bounces,visits,visit_duration\n2024-03-01,3,5,1,3,72000\n",
+ });
+
+ const bySession = new Map<string, string[]>();
+ for (const e of events) bySession.set(e.session_id, [...(bySession.get(e.session_id) ?? []), e.timestamp]);
+ const seconds = (ts: string) => {
+ const [h, m, s] = ts.slice(11).split(":").map(Number);
+ return h * 3600 + m * 60 + s;
+ };
+ const durations = [...bySession.values()].map(ts => seconds(ts.at(-1)!) - seconds(ts[0])).sort((a, b) => a - b);
+
+ // 1 bounce (0s) and 2 visits of 10 hours each. The session that would start
+ // at 16:00 starts at 13:59:59 instead, so it still ends before midnight.
+ expect(durations).toEqual([0, 36000, 36000]);
+ expect(durations.reduce((a, b) => a + b, 0)).toBe(72000);
+ });
+
it("is deterministic — the same zip twice yields byte-identical events", async () => {
const files = {
"pages.csv": BASIC_PAGES,
@@ -419,7 +439,7 @@ describe("referrers and utm parameters", () => {
});
describe("custom events", () => {
- it("attaches custom events to the day's sessions, spread across the day", async () => {
+ it("attaches custom events to the day's sessions at each session's last pageview", async () => {
const { events } = await runImport({
"pages.csv": BASIC_PAGES,
"custom_events.csv": "date,name,visitors,events,path,link_url\n2024-03-01,Signup,2,2,/signup,\n",
@@ -427,18 +447,34 @@ describe("custom events", () => {
const custom = events.filter(e => e.type === "custom_event");
expect(custom).toHaveLength(2);
+ // Session 1 has pageviews at 00:00:00 and 00:00:30; session 2 has one at
+ // 08:00:00. Neither event stretches its session.
expect(
custom.map(e => ({ timestamp: e.timestamp, event_name: e.event_name, pathname: e.pathname, props: e.props }))
).toEqual([
- { timestamp: "2024-03-01 00:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },
- { timestamp: "2024-03-01 12:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },
+ { timestamp: "2024-03-01 00:00:30", event_name: "Signup", pathname: "/signup", props: "{}" },
+ { timestamp: "2024-03-01 08:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },
]);
+ expect(custom[0].session_id).toBe(events[0].session_id);
+ expect(custom[1].session_id).toBe(events[1].session_id);
// They reuse pageview sessions, so the day's user ids do not multiply.
const pageviewSessions = new Set(events.filter(e => e.type === "pageview").map(e => e.session_id));
expect(custom.every(e => pageviewSessions.has(e.session_id))).toBe(true);
});
+ it("skips Plausible's internal engagement events", async () => {
+ const { events } = await runImport({
+ "pages.csv": BASIC_PAGES,
+ "custom_events.csv":
+ "date,name,visitors,events,path,link_url\n" +
+ "2024-03-01,engagement,3,40,,\n" +
+ "2024-03-01,Signup,1,1,/signup,\n",
+ });
+
+ expect(events.filter(e => e.type === "custom_event").map(e => e.event_name)).toEqual(["Signup"]);
+ });
+
it("carries link_url through as a url prop", async () => {
const { events } = await runImport({
"pages.csv": BASIC_PAGES,
diff --git a/client/src/lib/import/plausibleParser.ts b/client/src/lib/import/plausibleParser.ts
index d2c094c77..a3931e142 100644
--- a/client/src/lib/import/plausibleParser.ts
+++ b/client/src/lib/import/plausibleParser.ts
@@ -93,6 +93,9 @@ interface PoolDistributions {
const CHUNK_SIZE = 5000;
const SECONDS_IN_DAY = 86400;
const DEFAULT_PAGE_GAP_SECONDS = 30;
+// Plausible's own heartbeat for time-on-page and scroll depth. It is not a
+// user-defined goal, and replaying it as custom events stretches sessions.
+const PLAUSIBLE_INTERNAL_EVENTS = new Set(["engagement"]);
// Simple deterministic pseudo-random based on index
function deterministicPick<T>(
@@ -502,7 +505,9 @@ export class PlausibleCsvParser {
}
// Phase 5: Generate custom events, attached to existing sessions so they
- // share user_ids with pageview activity for the same day.
+ // share user_ids with pageview activity for the same day. Each event is
+ // stamped at its session's last pageview so it never extends the session
+ // beyond the duration Plausible reported.
const customEventsData = csvFiles.get("custom_events");
if (customEventsData) {
for (const row of customEventsData) {
@@ -518,12 +523,13 @@ export class PlausibleCsvParser {
const linkUrl = row.link_url || "";
const eventCount = parseInt(row.events || "0", 10);
if (eventCount <= 0 || !eventName) continue;
+ if (PLAUSIBLE_INTERNAL_EVENTS.has(eventName)) continue;
const propsStr = linkUrl ? JSON.stringify({ url: linkUrl }) : "{}";
for (let i = 0; i < eventCount; i++) {
const session = pool.sessions[i % pool.sessions.length];
- const offsetSeconds = Math.floor((i * SECONDS_IN_DAY) / eventCount);
+ const offsetSeconds = session.startSeconds + Math.max(0, session.pagesUsed - 1) * session.perPageSeconds;
const timestamp = formatTimestamp(date, offsetSeconds);
buffer.push({
@@ -643,6 +649,10 @@ export class PlausibleCsvParser {
userIds.push(generateUUID(dateSeed + i, dateSeed * 3 + i * 31337));
}
+ // Single-pageview visits contribute nothing to Plausible's visit_duration,
+ // so the day's total belongs to the non-bounced sessions alone.
+ const avgDurationSeconds = visitorsRowDuration > 0 && nonBounce > 0 ? visitorsRowDuration / nonBounce : 0;
+
const sessions: Session[] = [];
for (let i = 0; i < numSessions; i++) {
const isBounce = i < numBounces;
@@ -650,12 +660,17 @@ export class PlausibleCsvParser {
? 1
: Math.max(1, baseNonBounce + (i - numBounces < extraNonBounce ? 1 : 0));
- const startSeconds = Math.floor((i * SECONDS_IN_DAY) / numSessions);
- const avgDurationSeconds = visitorsRowDuration > 0 ? visitorsRowDuration / numSessions : 0;
const perPageSeconds =
- budget > 1
- ? Math.max(1, Math.floor(avgDurationSeconds / (budget - 1)) || DEFAULT_PAGE_GAP_SECONDS)
+ budget > 1 && avgDurationSeconds > 0
+ ? Math.max(1, Math.round(avgDurationSeconds / (budget - 1)))
: DEFAULT_PAGE_GAP_SECONDS;
+ // Start late sessions early enough to finish before midnight, so clamping
+ // to the end of the day does not cut their duration short.
+ const plannedSeconds = perPageSeconds * (budget - 1);
+ const startSeconds = Math.min(
+ Math.floor((i * SECONDS_IN_DAY) / numSessions),
+ Math.max(0, SECONDS_IN_DAY - 1 - plannedSeconds)
+ );
sessions.push({
sessionId: generateUUID(dateSeed * 7 + i, dateSeed * 11 + i * 7919),