Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

9.6 KB
Raw Blame
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { zipSync, strToU8 } from 'fflate'
import { PlausibleMapper as CloudPlausible } from '../apps/cloud/src/data-import/mappers/plausible.mapper'
import { PlausibleMapper as CommunityPlausible } from '../apps/community/src/data-import/mappers/plausible.mapper'
import { FathomMapper as CloudFathom } from '../apps/cloud/src/data-import/mappers/fathom.mapper'
import { FathomMapper as CommunityFathom } from '../apps/community/src/data-import/mappers/fathom.mapper'
import { SimpleAnalyticsMapper as CloudSimple } from '../apps/cloud/src/data-import/mappers/simple-analytics.mapper'
import { SimpleAnalyticsMapper as CommunitySimple } from '../apps/community/src/data-import/mappers/simple-analytics.mapper'
import { UmamiMapper as CloudUmami } from '../apps/cloud/src/data-import/mappers/umami.mapper'
import { UmamiMapper as CommunityUmami } from '../apps/community/src/data-import/mappers/umami.mapper'

const visitors =
  'date,visitors,pageviews,bounces,visits,visit_duration\n2024-01-01,2,5,0,2,30\n'
const collect = async (stream: AsyncIterable<any>) => {
  const rows = []
  for await (const row of stream) rows.push(row)
  return rows
}
let directory: string
beforeEach(() => {
  directory = fs.mkdtempSync(path.join(os.tmpdir(), 'import-test-'))
})
afterEach(() => {
  fs.rmSync(directory, { recursive: true, force: true })
})
function archive(files: Record<string, string>, level: 0 | 6 = 6) {
  const file = path.join(directory, 'export.zip')
  fs.writeFileSync(
    file,
    zipSync(
      Object.fromEntries(
        Object.entries(files).map(([name, data]) => [name, strToU8(data)]),
      ),
      { level },
    ),
  )
  return file
}

describe.each([
  ['cloud', CloudPlausible, CloudFathom, CloudSimple, CloudUmami],
  [
    'community',
    CommunityPlausible,
    CommunityFathom,
    CommunitySimple,
    CommunityUmami,
  ],
] as const)('%s imports', (_edition, Plausible, Fathom, Simple, Umami) => {
  it('imports all Plausible chunks, pageviews, dimensions and custom events', async () => {
    const file = archive({
      'export/imported_visitors_20240101_20240101.csv': visitors,
      'export/imported_visitors_20240102_20240102.csv': visitors.replaceAll(
        '2024-01-01',
        '2024-01-02',
      ),
      'imported_pages.csv':
        'date,hostname,page,pageviews\n2024-01-01,example.com,/a,5\n2024-01-02,example.com,/b,5\n',
      'imported_browsers.csv':
        'date,browser,browser_version,visits\n2024-01-01,Chrome,120,2\n',
      'imported_devices.csv': 'date,device,visits\n2024-01-01,Mobile,2\n',
      'imported_custom_events.csv': 'date,name,events\n2024-01-01,Signup,3\n',
    })
    const rows = await collect(
      new Plausible().createRowStream(file, 'testproject1', 7),
    )
    expect(rows.filter((row) => row.type === 'pageview')).toHaveLength(10)
    expect(rows.filter((row) => row.type === 'custom_event')).toHaveLength(3)
    expect(rows.find((row) => row.data.pg === '/a').data.br).toBe(
      'Mobile Chrome',
    )
    expect(rows.some((row) => row.data.pg === '/b')).toBe(true)
    expect(rows.every((row) => row.data.importID === 7)).toBe(true)
  })

  it('imports Plausible outbound destinations separately without losing counts', async () => {
    const file = archive({
      'imported_visitors.csv': visitors,
      'imported_custom_events.csv':
        'date,name,link_url,events\n2024-01-01,Outbound Link: Click,https://tickets.example.com/event,3\n2024-01-01,Outbound Link: Click,,2\n2024-01-01,Signup,,1\n',
    })
    const rows = await collect(
      new Plausible().createRowStream(file, 'testproject1', 7),
    )
    const outbound = rows.filter((row) => row.type === 'outbound')
    expect(outbound).toHaveLength(5)
    expect(
      outbound.filter(
        (row) => row.data.outbound_url === 'https://tickets.example.com/event',
      ),
    ).toHaveLength(3)
    expect(
      outbound.filter((row) => row.data.outbound_url === null),
    ).toHaveLength(2)
    expect(
      outbound.every((row) => row.data.importID === 7 && !row.data.event_name),
    ).toBe(true)
    expect(rows.filter((row) => row.type === 'custom_event')).toHaveLength(1)
  })

  it('handles a 57 MiB ZIP containing a CSV above the former 50 MiB limit', async () => {
    const padding = 'x'.repeat(2000)
    const record = `2024-01-01,example.com,/large,1,${padding}\n`
    const count = Math.ceil((57 * 1024 * 1024) / Buffer.byteLength(record))
    const file = archive(
      {
        'imported_visitors.csv': visitors,
        'imported_pages.csv': `date,hostname,page,pageviews,unused\n${record.repeat(count)}`,
      },
      0,
    )
    expect(fs.statSync(file).size).toBeGreaterThan(57 * 1024 * 1024)
    let timerRan = false
    const timer = setTimeout(() => {
      timerRan = true
    }, 0)
    try {
      const rows = await collect(
        new Plausible().createRowStream(file, 'testproject1', 1),
      )
      expect(rows).toHaveLength(5)
      expect(rows.every((row) => row.data.pg === '/large')).toBe(true)
      expect(timerRan).toBe(true)
    } finally {
      clearTimeout(timer)
    }
  })

  it('rejects unsupported ZIP contents and malformed CSV headers', async () => {
    await expect(
      collect(
        new Plausible().createRowStream(
          archive({ 'visitors.csv': visitors }),
          'testproject1',
          1,
        ),
      ),
    ).rejects.toThrow('imported_visitors')
    await expect(
      collect(
        new Plausible().createRowStream(
          archive({ 'imported_visitors.csv': 'date,wrong\n2024-01-01,1\n' }),
          'testproject1',
          1,
        ),
      ),
    ).rejects.toThrow('missing columns')
  })

  it('rejects truncated archives, excessive declared expansion, and invalid counts', async () => {
    const file = archive({ 'imported_visitors.csv': visitors })
    const bytes = fs.readFileSync(file)
    fs.writeFileSync(file, bytes.subarray(0, 60))
    await expect(
      collect(new Plausible().createRowStream(file, 'testproject1', 1)),
    ).rejects.toThrow()
    bytes.writeUInt32LE(2 * 1024 * 1024 * 1024 + 1, 22)
    fs.writeFileSync(file, bytes)
    await expect(
      collect(new Plausible().createRowStream(file, 'testproject1', 1)),
    ).rejects.toThrow('2 GiB')
    archive({
      'imported_visitors.csv': visitors.replace(',2,5,0,2,30', ',2,5.5,0,2,30'),
    })
    await expect(
      collect(new Plausible().createRowStream(file, 'testproject1', 1)),
    ).rejects.toThrow('invalid numeric')
  })

  it('rejects daily counts over the limits instead of silently truncating', async () => {
    const file = archive({
      'imported_visitors.csv': visitors.replace(
        ',2,5,0,2,30',
        ',2,5,0,1000001,30',
      ),
    })
    await expect(
      collect(new Plausible().createRowStream(file, 'testproject1', 1)),
    ).rejects.toThrow('daily event processing limits')
  })

  it('preserves days with custom events but no pageviews or visits', async () => {
    const file = archive({
      'imported_visitors.csv': visitors.replace(',2,5,0,2,30', ',1,0,0,0,0'),
      'imported_custom_events.csv': 'date,name,events\n2024-01-01,Signup,3\n',
    })
    const rows = await collect(
      new Plausible().createRowStream(file, 'testproject1', 1),
    )
    expect(rows).toHaveLength(3)
    expect(rows.every((row) => row.type === 'custom_event')).toBe(true)
  })

  it('cleans Plausible temporary files when the consumer stops or parsing fails', async () => {
    const cleanup = jest.spyOn(fs.promises, 'rm')
    const file = archive({ 'imported_visitors.csv': visitors })
    const rows = new Plausible().createRowStream(file, 'testproject1', 1)
    const stream = rows[Symbol.asyncIterator]()
    await stream.next()
    await stream.return!()
    expect(cleanup).toHaveBeenCalledWith(
      expect.stringContaining('swetrix-plausible-'),
      { recursive: true, force: true },
    )
    cleanup.mockClear()
    await expect(
      collect(
        new Plausible().createRowStream(
          archive({ 'bad.csv': 'invalid' }),
          'testproject1',
          1,
        ),
      ),
    ).rejects.toThrow()
    expect(cleanup).toHaveBeenCalled()
    cleanup.mockRestore()
  })

  it('reports missing CSV files without an unhandled stream error', async () => {
    for (const Mapper of [Fathom, Simple]) {
      await expect(
        collect(
          new Mapper().createRowStream(
            path.join(directory, 'missing.csv'),
            'testproject1',
            1,
          ),
        ),
      ).rejects.toThrow('Could not read')
    }
  })

  it('streams Fathom counts and Simple Analytics raw events', async () => {
    const file = path.join(directory, 'export.csv')
    fs.writeFileSync(
      file,
      'datetime,hostname,pathname,pageviews\n2024-01-01 00:00:00,example.com,/,3\n',
    )
    expect(
      await collect(new Fathom().createRowStream(file, 'testproject1', 1)),
    ).toHaveLength(3)
    fs.writeFileSync(
      file,
      'added_iso,datapoint,session_id,path\n2024-01-01T00:00:00Z,pageview,s1,/\n2024-01-01T00:01:00Z,Signup,s1,/\n',
    )
    const rows = await collect(
      new Simple().createRowStream(file, 'testproject1', 1),
    )
    expect(rows.map((row) => row.type)).toEqual(['pageview', 'custom_event'])
  })

  it('imports Umami BOM CSV and reports invalid ZIPs', async () => {
    const file = archive({
      'session.csv': 'ignored\n'.repeat(10000),
      'website_event.csv':
        '\uFEFFsession_id,created_at,event_type,url_path\ns1,2024-01-01 00:00:00,1,/\n',
    })
    expect(
      await collect(new Umami().createRowStream(file, 'testproject1', 1)),
    ).toHaveLength(1)
    fs.writeFileSync(file, 'invalid')
    await expect(
      collect(new Umami().createRowStream(file, 'testproject1', 1)),
    ).rejects.toThrow()
  })
})