Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

14.1 KB
Raw Blame
import Papa from "papaparse";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { ImportPlatform } from "@/types/import";

const authedFetch = vi.hoisted(() => vi.fn());
vi.mock("@/api/utils", () => ({ authedFetch }));

import { CsvParser } from "./csvParser";

// ---------------------------------------------------------------------------
// Harness
//
// startImport hands the input straight to Papa, which streams a string through
// the same chunk/complete callbacks (and pause/resume) a File would take, so a
// plain string stands in for the upload.
// ---------------------------------------------------------------------------

const SITE_ID = 42;
const IMPORT_ID = "import-9";

interface Upload {
  events: Record<string, string>[];
  isLastBatch: boolean;
}

function uploads(): Upload[] {
  return authedFetch.mock.calls.map(call => (call[2] as { data: Upload }).data);
}

const events = () => uploads().flatMap(u => u.events);

const shape = () => uploads().map(u => ({ n: u.events.length, last: u.isLastBatch }));

// Uploads settle on microtasks, so one macrotask lets every chunk and the final batch go out.
const flush = () => new Promise(resolve => setTimeout(resolve, 0));

function startImport(csv: string, platform: ImportPlatform = "umami"): CsvParser {
  const parser = new CsvParser(SITE_ID, IMPORT_ID, platform, "2024-01-01", "2024-12-31");
  parser.startImport(csv as unknown as File);
  return parser;
}

async function runImport(
  csv: string,
  opts: { platform?: ImportPlatform; earliest?: string; latest?: string } = {}
): Promise<Upload[]> {
  authedFetch.mockClear();
  const parser = new CsvParser(
    SITE_ID,
    IMPORT_ID,
    opts.platform ?? "umami",
    opts.earliest ?? "2024-01-01",
    opts.latest ?? "2024-12-31"
  );
  parser.startImport(csv as unknown as File);
  await flush();
  return uploads();
}

// The parser reads 5MB at a time; shrink that so a few rows span several chunks.
function useChunkSize(bytes: number) {
  const parse = Papa.parse;
  vi.spyOn(Papa, "parse").mockImplementation(((input: unknown, config: object) =>
    (parse as (input: unknown, config: object) => unknown)(input, {
      ...config,
      chunkSize: bytes,
    })) as typeof Papa.parse);
}

function deferred() {
  let resolve!: () => void;
  let reject!: (error: Error) => void;
  const promise = new Promise<void>((res, rej) => {
    resolve = res;
    reject = rej;
  });
  return { promise, resolve, reject };
}

const MANY_ROWS = "session_id,created_at\n" + [1, 2, 3, 4, 5].map(i => `sess-${i},2024-03-1${i} 10:00:00\n`).join("");

afterEach(() => {
  vi.restoreAllMocks();
});

const UMAMI_HEADER =
  "session_id,hostname,browser,os,device,screen,language,country,region,city," +
  "url_path,url_query,referrer_path,referrer_domain,page_title,event_type,event_name,distinct_id,created_at\n";

const UMAMI_ROW =
  "sess-1,example.com,chrome,Mac OS,desktop,1920x1080,en-US,US,US-CA,San Francisco," +
  "/pricing,?ref=hn,/post,news.ycombinator.com,Pricing,1,,user-7,2024-03-15 10:00:00\n";

const SA_HEADER =
  "added_iso,country_code,datapoint,document_referrer,hostname,lang_language,lang_region," +
  "path,query,screen_height,screen_width,session_id,user_agent,uuid\n";

const SA_ROW =
  "2024-03-15T10:00:00.000Z,US,pageview,https://news.ycombinator.com/,example.com,en,US," +
  "/pricing,?ref=hn,1080,1920,sess-1,Mozilla/5.0,uuid-1\n";

beforeEach(() => {
  authedFetch.mockReset();
  authedFetch.mockResolvedValue(undefined);
});

describe("upload envelope", () => {
  it("posts rows to the import's events endpoint and closes with an empty final batch", async () => {
    await runImport(UMAMI_HEADER + UMAMI_ROW);

    expect(authedFetch).toHaveBeenCalledTimes(2);
    expect(authedFetch.mock.calls[0][0]).toBe(`/sites/${SITE_ID}/imports/${IMPORT_ID}/events`);
    expect(authedFetch.mock.calls[0][2]).toMatchObject({ method: "POST" });
    expect(uploads().map(u => ({ n: u.events.length, last: u.isLastBatch }))).toEqual([
      { n: 1, last: false },
      { n: 0, last: true },
    ]);
  });

  it("sends only the final batch when nothing survives, never an empty data chunk", async () => {
    const sent = await runImport(UMAMI_HEADER);

    expect(sent).toEqual([{ events: [], isLastBatch: true }]);
  });
});

describe("umami rows", () => {
  it("maps every recognised column onto the wire shape", async () => {
    await runImport(UMAMI_HEADER + UMAMI_ROW);

    expect(events()[0]).toEqual({
      session_id: "sess-1",
      hostname: "example.com",
      browser: "chrome",
      os: "Mac OS",
      device: "desktop",
      screen: "1920x1080",
      language: "en-US",
      country: "US",
      region: "US-CA",
      city: "San Francisco",
      url_path: "/pricing",
      url_query: "?ref=hn",
      referrer_path: "/post",
      referrer_domain: "news.ycombinator.com",
      page_title: "Pricing",
      event_type: "1",
      event_name: "",
      distinct_id: "user-7",
      created_at: "2024-03-15 10:00:00",
    });
  });

  it("ignores columns it does not know about", async () => {
    await runImport("created_at,session_id,internal_id,tenant\n2024-03-15 10:00:00,sess-1,99,acme\n");

    expect(events()[0]).not.toHaveProperty("internal_id");
    expect(events()[0]).not.toHaveProperty("tenant");
    expect(events()[0]).toMatchObject({ session_id: "sess-1", created_at: "2024-03-15 10:00:00" });
  });

  it("drops rows with no created_at", async () => {
    const sent = await runImport("session_id,created_at\nsess-1,\nsess-2,2024-03-15 10:00:00\nsess-3,\n");

    expect(sent[0].events.map(e => e.session_id)).toEqual(["sess-2"]);
  });

  it("drops every row when the headers do not match the platform at all", async () => {
    const sent = await runImport("timestamp,page,visitor\n2024-03-15 10:00:00,/,v1\n");

    expect(sent).toEqual([{ events: [], isLastBatch: true }]);
  });
});

describe("simple analytics rows", () => {
  it("maps every recognised column onto the wire shape", async () => {
    await runImport(SA_HEADER + SA_ROW, { platform: "simple_analytics" });

    expect(events()[0]).toEqual({
      added_iso: "2024-03-15T10:00:00.000Z",
      country_code: "US",
      datapoint: "pageview",
      document_referrer: "https://news.ycombinator.com/",
      hostname: "example.com",
      lang_language: "en",
      lang_region: "US",
      path: "/pricing",
      query: "?ref=hn",
      screen_height: "1080",
      screen_width: "1920",
      session_id: "sess-1",
      user_agent: "Mozilla/5.0",
      uuid: "uuid-1",
    });
  });

  it("drops rows with no added_iso", async () => {
    const sent = await runImport("added_iso,uuid\n,uuid-1\n2024-03-15T10:00:00Z,uuid-2\n", {
      platform: "simple_analytics",
    });

    expect(sent[0].events.map(e => e.uuid)).toEqual(["uuid-2"]);
  });

  it("accepts ISO timestamps with an offset as well as Z", async () => {
    const sent = await runImport("added_iso,uuid\n2024-03-15T10:00:00+02:00,offset\n2024-03-15T10:00:00Z,zulu\n", {
      platform: "simple_analytics",
    });

    expect(sent[0].events.map(e => e.uuid)).toEqual(["offset", "zulu"]);
  });

  it("treats any platform other than umami as simple analytics", async () => {
    await runImport(SA_HEADER + SA_ROW, { platform: "plausible" });

    expect(events()[0]).toMatchObject({ added_iso: "2024-03-15T10:00:00.000Z", uuid: "uuid-1" });
  });
});

describe("date range", () => {
  it("keeps rows on the boundary days and drops rows outside them", async () => {
    const sent = await runImport(
      "session_id,created_at\n" +
        "before,2024-02-29 23:59:59\n" +
        "first,2024-03-01 00:00:00\n" +
        "last,2024-03-31 23:59:59\n" +
        "after,2024-04-01 00:00:00\n",
      { earliest: "2024-03-01", latest: "2024-03-31" }
    );

    expect(sent[0].events.map(e => e.session_id)).toEqual(["first", "last"]);
  });

  it("drops rows whose timestamp matches neither the umami nor the ISO shape", async () => {
    const sent = await runImport(
      "session_id,created_at\n" +
        "slash,15/03/2024 10:00:00\n" +
        "spaced-iso,2024-03-15 10:00:00.123\n" +
        "ok,2024-03-15 10:00:00\n"
    );

    // Note: a fractional-second Umami timestamp matches neither
    // "yyyy-MM-dd HH:mm:ss" nor Luxon's ISO parser (which needs the T), so it
    // is silently discarded.
    expect(sent[0].events.map(e => e.session_id)).toEqual(["ok"]);
  });

  it("uploads no rows, only closes the import, when the constructor gets an unparseable range", async () => {
    const parser = new CsvParser(SITE_ID, IMPORT_ID, "umami", "01-01-2024", "2024-12-31");
    parser.startImport((UMAMI_HEADER + UMAMI_ROW) as unknown as File);
    await flush();

    expect(uploads()).toEqual([{ events: [], isLastBatch: true }]);
  });

  it("uploads no rows, only closes the import, after cancel()", async () => {
    const parser = new CsvParser(SITE_ID, IMPORT_ID, "umami", "2024-01-01", "2024-12-31");
    parser.cancel();
    parser.startImport((UMAMI_HEADER + UMAMI_ROW) as unknown as File);
    await flush();

    expect(uploads()).toEqual([{ events: [], isLastBatch: true }]);
  });
});

describe("malformed and awkward csv", () => {
  it("keeps commas, escaped quotes and newlines inside quoted fields", async () => {
    await runImport(
      "session_id,page_title,created_at\n" +
        'sess-1,"Pricing, Plans and ""Add-ons""\nsecond line",2024-03-15 10:00:00\n'
    );

    expect(events()[0].page_title).toBe('Pricing, Plans and "Add-ons"\nsecond line');
  });

  it("strips a UTF-8 byte order mark from the first header", async () => {
    await runImport("session_id,created_at\nsess-1,2024-03-15 10:00:00\n");

    expect(events()).toHaveLength(1);
    expect(events()[0].session_id).toBe("sess-1");
  });

  it("handles CRLF line endings", async () => {
    await runImport("session_id,created_at\r\nsess-1,2024-03-15 10:00:00\r\n");

    expect(events()[0]).toMatchObject({ session_id: "sess-1", created_at: "2024-03-15 10:00:00" });
  });

  it("skips blank and whitespace-only lines", async () => {
    const sent = await runImport(
      "session_id,created_at\n\nsess-1,2024-03-15 10:00:00\n   \n\nsess-2,2024-03-16 10:00:00\n"
    );

    expect(sent[0].events.map(e => e.session_id)).toEqual(["sess-1", "sess-2"]);
  });

  it("tolerates rows with fewer columns than the header", async () => {
    const sent = await runImport(
      "session_id,hostname,created_at\nsess-1,example.com\nsess-2,example.com,2024-03-15 10:00:00\n"
    );

    // The short row has no created_at, so it is dropped rather than uploaded
    // with a missing timestamp.
    expect(sent[0].events.map(e => e.session_id)).toEqual(["sess-2"]);
  });

  it("tolerates rows with more columns than the header", async () => {
    await runImport("session_id,created_at\nsess-1,2024-03-15 10:00:00,extra,more\n");

    expect(events()[0]).toMatchObject({ session_id: "sess-1", created_at: "2024-03-15 10:00:00" });
  });

  it("uploads nothing for a completely empty input", async () => {
    const sent = await runImport("");

    expect(sent).toEqual([{ events: [], isLastBatch: true }]);
  });
});

describe("upload ordering", () => {
  it("holds the final batch until the last chunk's upload has landed", async () => {
    const upload = deferred();
    authedFetch.mockReturnValueOnce(upload.promise);

    startImport(UMAMI_HEADER + UMAMI_ROW);
    await flush();

    // The server marks the import complete on the final batch, so it must not overtake the data.
    expect(shape()).toEqual([{ n: 1, last: false }]);

    upload.resolve();
    await flush();

    expect(shape()).toEqual([
      { n: 1, last: false },
      { n: 0, last: true },
    ]);
  });

  it("uploads chunks one at a time, in file order", async () => {
    useChunkSize(30);
    let inFlight = 0;
    let maxInFlight = 0;
    authedFetch.mockImplementation(async () => {
      inFlight++;
      maxInFlight = Math.max(maxInFlight, inFlight);
      await flush();
      inFlight--;
    });

    startImport(MANY_ROWS);
    await vi.waitFor(() => expect(uploads().at(-1)?.isLastBatch).toBe(true));

    expect(maxInFlight).toBe(1);
    expect(uploads().filter(u => !u.isLastBatch).length).toBeGreaterThan(1);
    expect(events().map(e => e.session_id)).toEqual(["sess-1", "sess-2", "sess-3", "sess-4", "sess-5"]);
    expect(uploads().filter(u => u.isLastBatch)).toHaveLength(1);
  });
});

describe("upload failures", () => {
  beforeEach(() => {
    vi.spyOn(console, "error").mockImplementation(() => {});
  });

  it("closes the import only after the failed chunk has settled", async () => {
    const upload = deferred();
    authedFetch.mockReturnValueOnce(upload.promise);

    startImport(UMAMI_HEADER + UMAMI_ROW);
    await flush();
    expect(shape()).toEqual([{ n: 1, last: false }]);

    upload.reject(new Error("network down"));
    await flush();

    // The final batch is what lets the user delete the partial import, so a failure still sends it.
    expect(shape()).toEqual([
      { n: 1, last: false },
      { n: 0, last: true },
    ]);
  });

  it("stops reading the file at the first failed chunk", async () => {
    useChunkSize(30);
    authedFetch.mockRejectedValueOnce(new Error("network down"));

    startImport(MANY_ROWS);
    await flush();

    expect(events().map(e => e.session_id)).toEqual(["sess-1"]);
    expect(uploads().at(-1)).toEqual({ events: [], isLastBatch: true });
    expect(uploads()).toHaveLength(2);
  });

  it("stops reading the file when cancelled mid-upload, then closes the import", async () => {
    useChunkSize(30);
    const upload = deferred();
    authedFetch.mockReturnValueOnce(upload.promise);

    const parser = startImport(MANY_ROWS);
    await flush();
    parser.cancel();
    upload.resolve();
    await flush();

    expect(events().map(e => e.session_id)).toEqual(["sess-1"]);
    expect(uploads().at(-1)).toEqual({ events: [], isLastBatch: true });
    expect(uploads()).toHaveLength(2);
  });

  it("closes the import when the file can't be read", async () => {
    vi.spyOn(Papa, "parse").mockImplementation(((_input: unknown, config: { error: (error: Error) => void }) =>
      config.error(new Error("NotReadableError"))) as unknown as typeof Papa.parse);

    startImport(UMAMI_HEADER + UMAMI_ROW);
    await flush();

    expect(uploads()).toEqual([{ events: [], isLastBatch: true }]);
  });
});