Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

20.6 KB
Raw Blame
import JSZip from "jszip";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";

const authedFetch = vi.hoisted(() => vi.fn());
vi.mock("@/api/utils", () => ({ authedFetch }));

import { PlausibleCsvParser } from "./plausibleParser";

// ---------------------------------------------------------------------------
// Harness
//
// Only the class is exported, so every helper (file identification, dimension
// normalisation, session synthesis, chunking) is exercised through a real ZIP
// fed to startImport, with the upload seam mocked. JSZip accepts a Uint8Array,
// so no browser File is needed.
// ---------------------------------------------------------------------------

const SITE_ID = 7;
const IMPORT_ID = "import-1";

type Csvs = Record<string, string>;

interface Upload {
  events: SyntheticEvent[];
  isLastBatch: boolean;
}

interface SyntheticEvent {
  timestamp: string;
  session_id: string;
  user_id: string;
  hostname: string;
  pathname: string;
  querystring: string;
  referrer: string;
  browser: string;
  browser_version: string;
  operating_system: string;
  operating_system_version: string;
  device_type: string;
  country: string;
  region: string;
  city: string;
  type: string;
  event_name: string;
  props: string;
}

async function zipOf(files: Csvs): Promise<File> {
  const zip = new JSZip();
  for (const [name, content] of Object.entries(files)) zip.file(name, content);
  const bytes = await zip.generateAsync({ type: "uint8array" });
  return bytes as unknown as File;
}

async function runImport(
  files: Csvs,
  range: { earliest?: string; latest?: string } = {}
): Promise<{ uploads: Upload[]; events: SyntheticEvent[] }> {
  // Recorded calls, but not the mock implementation — a test may have armed a
  // rejection before calling in.
  authedFetch.mockClear();
  const parser = new PlausibleCsvParser(
    SITE_ID,
    IMPORT_ID,
    range.earliest ?? "2024-01-01",
    range.latest ?? "2024-12-31"
  );
  await parser.startImport(await zipOf(files));
  return { uploads: uploads(), events: uploads().flatMap(u => u.events) };
}

function uploads(): Upload[] {
  return authedFetch.mock.calls.map(call => (call[2] as { data: Upload }).data);
}

// A two-page day with no visitors.csv. The pages totals alone (4 pageviews
// across 3 visits) size the session pool, which makes every downstream number
// in these tests hand-checkable.
const BASIC_PAGES =
  "date,page,hostname,visitors,pageviews,visits\n" +
  "2024-03-01,/,example.com,3,3,2\n" +
  "2024-03-01,/about,example.com,1,1,1\n";

beforeEach(() => {
  authedFetch.mockReset();
  authedFetch.mockResolvedValue(undefined);
});

afterEach(() => {
  vi.restoreAllMocks();
});

describe("upload envelope", () => {
  it("posts each chunk to the import's events endpoint and closes with an empty final batch", async () => {
    await runImport({ "pages.csv": BASIC_PAGES });

    expect(authedFetch).toHaveBeenCalledTimes(2);
    expect(authedFetch.mock.calls[0][0]).toBe(`/sites/${SITE_ID}/imports/${IMPORT_ID}/events`);
    expect(authedFetch.mock.calls[0][2]).toMatchObject({ method: "POST" });
    expect(uploads().map(u => ({ n: u.events.length, last: u.isLastBatch }))).toEqual([
      { n: 4, last: false },
      { n: 0, last: true },
    ]);
  });
});

describe("file identification", () => {
  it("finalises without events when the zip has no pages.csv", async () => {
    const { uploads } = await runImport({
      "browsers.csv": "date,browser,browser_version,visitors,pageviews\n2024-03-01,Chrome,120,5,5\n",
    });

    expect(uploads).toEqual([{ events: [], isLastBatch: true }]);
  });

  it("finalises without events when pages.csv has a header but no rows", async () => {
    const { uploads } = await runImport({ "pages.csv": "date,page,hostname,visitors,pageviews,visits\n" });

    expect(uploads).toEqual([{ events: [], isLastBatch: true }]);
  });

  it("ignores non-csv entries and csvs with unrecognised headers", async () => {
    const { events } = await runImport({
      "README.txt": "not a csv at all",
      "mystery.csv": "alpha,beta\n1,2\n",
      "nested/pages.csv": BASIC_PAGES,
    });

    expect(events).toHaveLength(4);
  });
});

describe("pageview synthesis", () => {
  it("spreads pageviews over a session pool sized by the pages totals", async () => {
    const { events } = await runImport({ "pages.csv": BASIC_PAGES });

    // 3 visits => 3 sessions, starting at even thirds of the day. 4 pageviews
    // are dealt round-robin, so the fourth lands back on the first session,
    // one page-gap (30s) after its first pageview.
    expect(events.map(e => ({ timestamp: e.timestamp, pathname: e.pathname }))).toEqual([
      { timestamp: "2024-03-01 00:00:00", pathname: "/" },
      { timestamp: "2024-03-01 08:00:00", pathname: "/" },
      { timestamp: "2024-03-01 16:00:00", pathname: "/" },
      { timestamp: "2024-03-01 00:00:30", pathname: "/about" },
    ]);
    expect(new Set(events.map(e => e.session_id)).size).toBe(3);
    expect(events[0].session_id).toBe(events[3].session_id);
    expect(events[0].user_id).toBe(events[3].user_id);
  });

  it("stamps every pageview with type pageview, no event name and empty props", async () => {
    const { events } = await runImport({ "pages.csv": BASIC_PAGES });

    expect(events.every(e => e.type === "pageview" && e.event_name === "" && e.props === "{}")).toBe(true);
  });

  it("takes hostname and pathname from the page row, defaulting a blank page to /", async () => {
    const { events } = await runImport({
      "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,,blog.example.com,1,1,1\n",
    });

    expect(events).toHaveLength(1);
    expect(events[0]).toMatchObject({ hostname: "blog.example.com", pathname: "/" });
  });

  it("skips rows with no date, no pageviews or a non-numeric pageview count", async () => {
    const { events } = await runImport({
      "pages.csv":
        "date,page,hostname,visitors,pageviews,visits\n" +
        ",/orphan,example.com,1,5,1\n" +
        "2024-03-01,/zero,example.com,0,0,1\n" +
        "2024-03-01,/junk,example.com,1,abc,1\n" +
        "2024-03-01,/real,example.com,1,1,1\n",
    });

    expect(events.map(e => e.pathname)).toEqual(["/real"]);
  });

  it("gives bounced sessions exactly one pageview when visitors.csv reports bounces", async () => {
    const { events } = await runImport({
      "pages.csv":
        "date,page,hostname,visitors,pageviews,visits\n" +
        "2024-03-01,/,example.com,2,3,2\n" +
        "2024-03-01,/about,example.com,1,1,1\n",
      "visitors.csv": "date,visitors,pageviews,bounces,visits,visit_duration\n2024-03-01,2,4,1,2,120\n",
    });

    // visitors.csv wins over the pages totals: 2 visits => 2 sessions, 1 of
    // which bounced and therefore gets a single pageview.
    const perSession = new Map<string, number>();
    for (const e of events) perSession.set(e.session_id, (perSession.get(e.session_id) ?? 0) + 1);
    expect([...perSession.values()].sort()).toEqual([1, 3]);
    expect(events).toHaveLength(4);
  });

  it("gives the day's visit_duration to the non-bounced sessions, without truncating late ones", async () => {
    const { events } = await runImport({
      "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,3,5,3\n",
      "visitors.csv": "date,visitors,pageviews,bounces,visits,visit_duration\n2024-03-01,3,5,1,3,72000\n",
    });

    const bySession = new Map<string, string[]>();
    for (const e of events) bySession.set(e.session_id, [...(bySession.get(e.session_id) ?? []), e.timestamp]);
    const seconds = (ts: string) => {
      const [h, m, s] = ts.slice(11).split(":").map(Number);
      return h * 3600 + m * 60 + s;
    };
    const durations = [...bySession.values()].map(ts => seconds(ts.at(-1)!) - seconds(ts[0])).sort((a, b) => a - b);

    // 1 bounce (0s) and 2 visits of 10 hours each. The session that would start
    // at 16:00 starts at 13:59:59 instead, so it still ends before midnight.
    expect(durations).toEqual([0, 36000, 36000]);
    expect(durations.reduce((a, b) => a + b, 0)).toBe(72000);
  });

  it("is deterministic — the same zip twice yields byte-identical events", async () => {
    const files = {
      "pages.csv": BASIC_PAGES,
      "browsers.csv":
        "date,browser,browser_version,visitors,pageviews\n2024-03-01,Chrome,120,2,2\n2024-03-01,Firefox,124,1,1\n",
      "locations.csv":
        "date,country,region,city,visitors,pageviews\n2024-03-01,US,US-CA,5391959,2,2\n2024-03-01,DE,DE-BE,2950159,1,1\n",
    };

    const first = await runImport(files);
    const second = await runImport(files);

    expect(second.events).toEqual(first.events);
  });

  it("assigns pages by entry weight when entry_pages.csv is present", async () => {
    const withoutEntryPages = await runImport({ "pages.csv": BASIC_PAGES });
    const withEntryPages = await runImport({
      "pages.csv": BASIC_PAGES,
      "entry_pages.csv": "date,entry_page,visitors,entrances\n2024-03-01,/about,50,50\n2024-03-01,/,1,1\n",
    });

    // Same events, but /about is dealt first so it starts a session instead of
    // continuing one.
    expect(withEntryPages.events).toHaveLength(4);
    expect(withEntryPages.events.map(e => e.pathname)).toEqual(["/about", "/", "/", "/"]);
    expect(withoutEntryPages.events.map(e => e.pathname)).toEqual(["/", "/", "/", "/about"]);
    expect(withEntryPages.events[0].timestamp).toBe("2024-03-01 00:00:00");
  });
});

describe("date range", () => {
  it("drops dates outside the allowed window, inclusive of both ends", async () => {
    const { events } = await runImport(
      {
        "pages.csv":
          "date,page,hostname,visitors,pageviews,visits\n" +
          "2024-02-28,/early,example.com,1,1,1\n" +
          "2024-03-01,/first,example.com,1,1,1\n" +
          "2024-03-02,/last,example.com,1,1,1\n" +
          "2024-03-03,/late,example.com,1,1,1\n",
      },
      { earliest: "2024-03-01", latest: "2024-03-02" }
    );

    expect(events.map(e => e.pathname)).toEqual(["/first", "/last"]);
  });

  it("drops rows whose date is not yyyy-MM-dd", async () => {
    const { events } = await runImport({
      "pages.csv":
        "date,page,hostname,visitors,pageviews,visits\n" +
        "03/01/2024,/us-format,example.com,1,1,1\n" +
        "2024-03-01,/iso,example.com,1,1,1\n",
    });

    expect(events.map(e => e.pathname)).toEqual(["/iso"]);
  });

  it("uploads nothing at all when the constructor gets an unparseable range", async () => {
    const parser = new PlausibleCsvParser(SITE_ID, IMPORT_ID, "not-a-date", "2024-12-31");
    await parser.startImport(await zipOf({ "pages.csv": BASIC_PAGES }));

    expect(authedFetch).not.toHaveBeenCalled();
  });

  it("uploads nothing after cancel()", async () => {
    const parser = new PlausibleCsvParser(SITE_ID, IMPORT_ID, "2024-01-01", "2024-12-31");
    parser.cancel();
    await parser.startImport(await zipOf({ "pages.csv": BASIC_PAGES }));

    expect(authedFetch).not.toHaveBeenCalled();
  });
});

describe("dimension normalisation", () => {
  const single = (extra: Csvs) => ({
    "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,1,1,1\n",
    ...extra,
  });

  it("maps Plausible's browser, os and device names onto Rybbit's", async () => {
    const { events } = await runImport(
      single({
        "browsers.csv": "date,browser,browser_version,visitors,pageviews\n2024-03-01,Microsoft Edge,120.0,1,1\n",
        "operating_systems.csv":
          "date,operating_system,operating_system_version,visitors,pageviews\n2024-03-01,GNU/Linux,6.5,1,1\n",
        "devices.csv": "date,device,visitors,pageviews\n2024-03-01,laptop,1,1\n",
      })
    );

    expect(events[0]).toMatchObject({
      browser: "Edge",
      browser_version: "120.0",
      operating_system: "Linux",
      operating_system_version: "6.5",
      device_type: "Desktop",
    });
  });

  it.each([
    ["Microsoft Edge", "Edge"],
    ["Samsung Browser", "Samsung Internet"],
    ["Yandex Browser", "Yandex"],
    ["MICROSOFT EDGE", "Edge"],
    ["Firefox", "Firefox"],
    ["", ""],
  ])("normalises browser %s to %s", async (input, expected) => {
    const { events } = await runImport(
      single({ "browsers.csv": `date,browser,browser_version,visitors,pageviews\n2024-03-01,${input},1.0,1,1\n` })
    );
    expect(events[0].browser).toBe(expected);
  });

  it.each([
    ["GNU/Linux", "Linux"],
    ["Ubuntu", "Linux"],
    ["Mac", "macOS"],
    ["Mac OS X", "macOS"],
    ["iOS", "iOS"],
    ["iPadOS", "iOS"],
    ["Android", "Android"],
    ["Windows", "Windows"],
    ["ChromeOS", "Chrome OS"],
    ["Chrome OS", "Chrome OS"],
    ["FreeBSD", "FreeBSD"],
  ])("normalises operating system %s to %s", async (input, expected) => {
    const { events } = await runImport(
      single({
        "operating_systems.csv": `date,operating_system,operating_system_version,visitors,pageviews\n2024-03-01,${input},1,1,1\n`,
      })
    );
    expect(events[0].operating_system).toBe(expected);
  });

  it.each([
    ["desktop", "Desktop"],
    ["laptop", "Desktop"],
    ["Desktop", "Desktop"],
    ["mobile", "Mobile"],
    ["tablet", "Mobile"],
    ["wearable", "wearable"],
  ])("normalises device %s to %s", async (input, expected) => {
    const { events } = await runImport(
      single({ "devices.csv": `date,device,visitors,pageviews\n2024-03-01,${input},1,1\n` })
    );
    expect(events[0].device_type).toBe(expected);
  });

  it("keeps country and region but drops Plausible's numeric city id", async () => {
    const { events } = await runImport(
      single({ "locations.csv": "date,country,region,city,visitors,pageviews\n2024-03-01,US,US-CA,5391959,1,1\n" })
    );

    expect(events[0]).toMatchObject({ country: "US", region: "US-CA", city: "" });
  });

  it("leaves every dimension blank when the zip carries only pages.csv", async () => {
    const { events } = await runImport(single({}));

    expect(events[0]).toMatchObject({
      browser: "",
      browser_version: "",
      operating_system: "",
      operating_system_version: "",
      device_type: "",
      country: "",
      region: "",
      city: "",
      referrer: "",
      querystring: "",
    });
  });
});

describe("referrers and utm parameters", () => {
  const withSource = async (row: string) => {
    const { events } = await runImport({
      "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,1,1,1\n",
      "sources.csv":
        "date,source,referrer,utm_source,utm_medium,utm_campaign,utm_content,utm_term,visitors,pageviews\n" + row,
    });
    return events[0];
  };

  it("promotes a bare referrer host to https", async () => {
    const event = await withSource("2024-03-01,Google,google.com/search,,,,,,1,1\n");
    expect(event.referrer).toBe("https://google.com/search");
  });

  it("falls back to the source when there is no referrer", async () => {
    expect((await withSource("2024-03-01,t.co,,,,,,,1,1\n")).referrer).toBe("https://t.co");
  });

  it("drops friendly source names that cannot become urls", async () => {
    expect((await withSource("2024-03-01,Brave,,,,,,,1,1\n")).referrer).toBe("");
    expect((await withSource("2024-03-01,Direct / None,,,,,,,1,1\n")).referrer).toBe("https://Direct / None");
  });

  it("keeps a value that already has a scheme", async () => {
    expect((await withSource("2024-03-01,android-app://com.linkedin.android,,,,,,,1,1\n")).referrer).toBe(
      "android-app://com.linkedin.android"
    );
  });

  it("rebuilds a querystring from the utm columns in a fixed order", async () => {
    const event = await withSource("2024-03-01,newsletter,,spring,email,launch,banner,shoes,1,1\n");

    expect(event.querystring).toBe(
      "?utm_source=spring&utm_medium=email&utm_campaign=launch&utm_content=banner&utm_term=shoes"
    );
  });

  it("omits empty utm columns and url-encodes the rest", async () => {
    const event = await withSource("2024-03-01,newsletter,,spring sale,,,,,1,1\n");

    expect(event.querystring).toBe("?utm_source=spring+sale");
  });

  it("leaves the querystring empty when no utm columns are set", async () => {
    expect((await withSource("2024-03-01,Google,google.com,,,,,,1,1\n")).querystring).toBe("");
  });
});

describe("custom events", () => {
  it("attaches custom events to the day's sessions at each session's last pageview", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv": "date,name,visitors,events,path,link_url\n2024-03-01,Signup,2,2,/signup,\n",
    });

    const custom = events.filter(e => e.type === "custom_event");
    expect(custom).toHaveLength(2);
    // Session 1 has pageviews at 00:00:00 and 00:00:30; session 2 has one at
    // 08:00:00. Neither event stretches its session.
    expect(
      custom.map(e => ({ timestamp: e.timestamp, event_name: e.event_name, pathname: e.pathname, props: e.props }))
    ).toEqual([
      { timestamp: "2024-03-01 00:00:30", event_name: "Signup", pathname: "/signup", props: "{}" },
      { timestamp: "2024-03-01 08:00:00", event_name: "Signup", pathname: "/signup", props: "{}" },
    ]);
    expect(custom[0].session_id).toBe(events[0].session_id);
    expect(custom[1].session_id).toBe(events[1].session_id);

    // They reuse pageview sessions, so the day's user ids do not multiply.
    const pageviewSessions = new Set(events.filter(e => e.type === "pageview").map(e => e.session_id));
    expect(custom.every(e => pageviewSessions.has(e.session_id))).toBe(true);
  });

  it("skips Plausible's internal engagement events", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv":
        "date,name,visitors,events,path,link_url\n" +
        "2024-03-01,engagement,3,40,,\n" +
        "2024-03-01,Signup,1,1,/signup,\n",
    });

    expect(events.filter(e => e.type === "custom_event").map(e => e.event_name)).toEqual(["Signup"]);
  });

  it("carries link_url through as a url prop", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv":
        "date,name,visitors,events,path,link_url\n2024-03-01,Outbound Link: Click,1,1,/,https://x.com/rybbit\n",
    });

    expect(events.filter(e => e.type === "custom_event")[0].props).toBe('{"url":"https://x.com/rybbit"}');
  });

  it("uses the session hostname rather than a per-row one", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv": "date,name,visitors,events,path,link_url\n2024-03-01,Signup,1,1,/signup,\n",
    });

    expect(events.filter(e => e.type === "custom_event")[0].hostname).toBe("example.com");
  });

  it("defaults a blank path to /", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv": "date,name,visitors,events,path,link_url\n2024-03-01,Signup,1,1,,\n",
    });

    expect(events.filter(e => e.type === "custom_event")[0].pathname).toBe("/");
  });

  it("skips rows with no name, no events or a date that has no pageviews", async () => {
    const { events } = await runImport({
      "pages.csv": BASIC_PAGES,
      "custom_events.csv":
        "date,name,visitors,events,path,link_url\n" +
        "2024-03-01,,1,5,/anon,\n" +
        "2024-03-01,ZeroEvents,1,0,/zero,\n" +
        "2024-03-09,OtherDay,1,3,/other,\n" +
        "2024-03-01,Kept,1,1,/kept,\n",
    });

    expect(events.filter(e => e.type === "custom_event").map(e => e.pathname)).toEqual(["/kept"]);
  });
});

describe("chunking and timestamp clamping", () => {
  it("uploads in 5000-event chunks and clamps timestamps to the end of the day", async () => {
    const { uploads, events } = await runImport({
      "pages.csv": "date,page,hostname,visitors,pageviews,visits\n2024-03-01,/,example.com,1,12000,1\n",
    });

    expect(uploads.map(u => ({ n: u.events.length, last: u.isLastBatch }))).toEqual([
      { n: 5000, last: false },
      { n: 5000, last: false },
      { n: 2000, last: false },
      { n: 0, last: true },
    ]);

    // One session, 30s between pages: everything past 23:59:59 is pinned there
    // rather than spilling into the next day.
    expect(events[0].timestamp).toBe("2024-03-01 00:00:00");
    expect(events[2879].timestamp).toBe("2024-03-01 23:59:30");
    expect(events[2880].timestamp).toBe("2024-03-01 23:59:59");
    expect(events.at(-1)!.timestamp).toBe("2024-03-01 23:59:59");
  });
});

describe("upload failures", () => {
  it("swallows the error and still attempts to finalise the import", async () => {
    vi.spyOn(console, "error").mockImplementation(() => {});
    authedFetch.mockRejectedValue(new Error("network down"));

    await expect(runImport({ "pages.csv": BASIC_PAGES })).resolves.toBeDefined();

    expect(authedFetch).toHaveBeenCalledTimes(2);
    expect(uploads()[1]).toEqual({ events: [], isLastBatch: true });
  });
});