Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

31.5 KB
Raw Blame
import * as fs from 'fs'
import * as path from 'path'
import * as crypto from 'crypto'
import * as os from 'os'
import { Unzip, UnzipInflate } from 'fflate'
import { createInterface } from 'readline'
import { readCsvRows } from './csv.utils'

import {
  ImportMapper,
  ImportError,
  AnalyticsImportRow,
} from './mapper.interface'
import {
  normalizeNull,
  truncate,
  sessionIdToPsid,
  MOBILE_BROWSER_VARIANTS,
} from './mapper.utils'

const MAX_PLAUSIBLE_ZIP_BYTES = 100 * 1024 * 1024
const MAX_PLAUSIBLE_UNCOMPRESSED_BYTES = 2 * 1024 * 1024 * 1024
const MAX_DAILY_BYTES = 128 * 1024 * 1024
const MIN_PAGE_GAP_SEC = 5
const MAX_PAGE_GAP_SEC = 30 * 60
const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/

const MAX_SESSIONS_PER_DAY = 1_000_000
const MAX_PAGEVIEWS_PER_SESSION = 1_000
const MAX_EVENTS_PER_DAY = 1_000_000
const MAX_GENERATED_ROWS_PER_IMPORT = 100_000_000
const MAX_SAFE_COUNT = Number.MAX_SAFE_INTEGER

const FILE_PATTERNS: Record<string, RegExp> = {
  visitors: /(^|\/)imported_visitors[^/]*\.csv$/i,
  pages: /(^|\/)imported_pages[^/]*\.csv$/i,
  browsers: /(^|\/)imported_browsers[^/]*\.csv$/i,
  devices: /(^|\/)imported_devices[^/]*\.csv$/i,
  operatingSystems: /(^|\/)imported_operating_systems[^/]*\.csv$/i,
  locations: /(^|\/)imported_locations[^/]*\.csv$/i,
  sources: /(^|\/)imported_sources[^/]*\.csv$/i,
  entryPages: /(^|\/)imported_entry_pages[^/]*\.csv$/i,
  exitPages: /(^|\/)imported_exit_pages[^/]*\.csv$/i,
  customEvents: /(^|\/)imported_custom_events[^/]*\.csv$/i,
}

const OS_MAP: Record<string, string> = {
  Mac: 'macOS',
  GNU: 'Linux',
  Ubuntu: 'Linux',
  iOS: 'iOS',
  Android: 'Android',
  Windows: 'Windows',
  'Chrome OS': 'Chrome OS',
}

const DEVICE_MAP: Record<string, string> = {
  Desktop: 'desktop',
  Laptop: 'desktop',
  Mobile: 'mobile',
  Tablet: 'tablet',
}

interface VisitorRow {
  date: string
  visitors: number
  pageviews: number
  visits: number
  bounces: number
  visit_duration: number
}

interface BrowserRow {
  browser: string | null
  browser_version: string | null
  visits: number
}

interface DeviceRow {
  device: string | null
  visits: number
}

interface OsRow {
  operating_system: string | null
  operating_system_version: string | null
  visits: number
}

interface LocationRow {
  country: string | null
  region: string | null
  visits: number
}

interface SourceRow {
  source: string | null
  referrer: string | null
  utm_source: string | null
  utm_medium: string | null
  utm_campaign: string | null
  utm_content: string | null
  utm_term: string | null
  visits: number
}

interface PageRow {
  hostname: string | null
  page: string | null
  pageviews: number
}

interface EntryPageRow {
  entry_page: string | null
  entrances: number
}

interface ExitPageRow {
  exit_page: string | null
  exits: number
}

interface CustomEventRow {
  name: string
  url: string | null
  events: number
}

interface DailyData {
  visitors: VisitorRow
  browsers: BrowserRow[]
  devices: DeviceRow[]
  os: OsRow[]
  locations: LocationRow[]
  sources: SourceRow[]
  pages: PageRow[]
  entryPages: EntryPageRow[]
  exitPages: ExitPageRow[]
  customEvents: CustomEventRow[]
}

function seededPrng(seed: number): () => number {
  let s = seed >>> 0
  return () => {
    s = (s + 0x6d2b79f5) >>> 0
    let t = s
    t = Math.imul(t ^ (t >>> 15), t | 1)
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296
  }
}

function seedFromString(s: string): number {
  return crypto.createHash('sha256').update(s).digest().readUInt32BE(0)
}

function toInt(value: string | undefined, fieldName = 'value'): number {
  if (value === undefined || value === null || value === '') return 0
  const n = /^\d+$/.test(value) ? Number(value) : NaN
  if (!Number.isFinite(n) || Number.isNaN(n) || !Number.isSafeInteger(n)) {
    throw new ImportError(
      `Plausible export contains an invalid numeric ${fieldName}: ${value}`,
    )
  }
  if (n < 0) {
    throw new ImportError(
      `Plausible export contains a negative numeric ${fieldName}: ${value}`,
    )
  }
  return Math.min(n, MAX_SAFE_COUNT)
}

async function* readDailyRows(
  paths: string[],
): AsyncIterable<Record<string, string>> {
  for (const filePath of paths) {
    const input = fs.createReadStream(filePath)
    const lines = createInterface({ input, crlfDelay: Infinity })
    input.on('error', (error) => lines.emit('error', error))
    try {
      for await (const line of lines) {
        yield JSON.parse(line)
      }
    } finally {
      lines.close()
      input.destroy()
    }
  }
}

function mapBrowser(raw: string | null, device: string | null): string | null {
  if (!raw) return null
  const isMobile = device === 'mobile' || device === 'tablet'
  if (isMobile && MOBILE_BROWSER_VARIANTS[raw]) {
    return MOBILE_BROWSER_VARIANTS[raw]
  }
  return raw
}

function mapOS(raw: string | null): string | null {
  if (!raw) return null
  return OS_MAP[raw] ?? raw
}

function mapDevice(raw: string | null): string {
  if (!raw) return 'desktop'
  return DEVICE_MAP[raw] ?? raw.toLowerCase()
}

function parseRegion(raw: string | null): string | null {
  if (!raw) return null
  const dash = raw.indexOf('-')
  if (dash < 0) return raw.slice(0, 10)
  return raw.slice(dash + 1).slice(0, 10) || null
}

function buildReferrer(
  referrer: string | null,
  source: string | null,
  hostname: string | null,
): string | null {
  const candidate = referrer || source
  if (!candidate || candidate.toLowerCase() === 'direct') return null

  let url: string
  if (candidate.includes('://')) {
    url = candidate
  } else if (candidate.includes('.')) {
    url = `https://${candidate}`
  } else {
    return null
  }

  if (hostname) {
    try {
      const parsed = new URL(url)
      const cleanRef = parsed.hostname.replace(/^www\./, '')
      const cleanHost = hostname.replace(/^www\./, '')
      if (cleanRef === cleanHost) return null
    } catch {
      // ignore
    }
  }

  return url
}

function pickSiteHostname(pages: PageRow[]): string | null {
  if (pages.length === 0) return null
  const counts = new Map<string, number>()
  for (const p of pages) {
    if (!p.hostname) continue
    counts.set(p.hostname, (counts.get(p.hostname) ?? 0) + p.pageviews)
  }
  let best: string | null = null
  let bestCount = -1
  for (const [host, count] of counts) {
    if (count > bestCount) {
      best = host
      bestCount = count
    }
  }
  return best
}

const MAX_POOL_SIZE = 2_000_000

function buildShuffledPool<T>(
  items: T[],
  counts: number[],
  prng: () => number,
): T[] {
  if (items.length === 0) return []

  const safeCounts: number[] = new Array(items.length)
  let total = 0
  for (let i = 0; i < items.length; i++) {
    const c = counts[i]
    const safe = c > 0 ? c : 0
    safeCounts[i] = safe
    total += safe
  }
  if (total <= 0) return []

  // Allocate per-item slots, scaling proportionally if the total would
  // exceed MAX_POOL_SIZE. Largest-remainder method keeps the assigned
  // counts as close to the original ratios as possible while preserving
  // the exact target total.
  let allocations: number[]
  let poolSize: number
  if (total > MAX_POOL_SIZE) {
    allocations = new Array(items.length).fill(0)
    const fractions: { idx: number; rem: number }[] = []
    let assigned = 0
    for (let i = 0; i < items.length; i++) {
      if (safeCounts[i] === 0) continue
      const scaled = (safeCounts[i] * MAX_POOL_SIZE) / total
      const floor = Math.floor(scaled)
      allocations[i] = floor
      assigned += floor
      fractions.push({ idx: i, rem: scaled - floor })
    }
    fractions.sort((a, b) => b.rem - a.rem || a.idx - b.idx)
    let remaining = MAX_POOL_SIZE - assigned
    for (let i = 0; i < fractions.length && remaining > 0; i++) {
      allocations[fractions[i].idx] += 1
      remaining -= 1
    }
    poolSize = MAX_POOL_SIZE
  } else {
    allocations = safeCounts
    poolSize = total
  }

  const pool: T[] = new Array(poolSize)
  let p = 0
  for (let i = 0; i < items.length; i++) {
    const c = allocations[i]
    for (let j = 0; j < c; j++) {
      pool[p++] = items[i]
    }
  }

  for (let i = pool.length - 1; i > 0; i--) {
    const j = Math.floor(prng() * (i + 1))
    const tmp = pool[i]
    pool[i] = pool[j]
    pool[j] = tmp
  }

  return pool
}

function pickFromPool<T>(pool: T[], idx: number): T | null {
  if (pool.length === 0) return null
  return pool[idx % pool.length]
}

interface PoolBundle {
  browsers: BrowserRow[]
  devices: DeviceRow[]
  os: OsRow[]
  locations: LocationRow[]
  sources: SourceRow[]
  pages: PageRow[]
  entryPages: EntryPageRow[]
  exitPages: ExitPageRow[]
}

function buildPools(
  daily: DailyData,
  importID: number,
  date: string,
): PoolBundle {
  const seedFor = (label: string) =>
    seededPrng(seedFromString(`${importID}|${date}|pool|${label}`))

  return {
    browsers: buildShuffledPool(
      daily.browsers,
      daily.browsers.map((r) => r.visits),
      seedFor('browsers'),
    ),
    devices: buildShuffledPool(
      daily.devices,
      daily.devices.map((r) => r.visits),
      seedFor('devices'),
    ),
    os: buildShuffledPool(
      daily.os,
      daily.os.map((r) => r.visits),
      seedFor('os'),
    ),
    locations: buildShuffledPool(
      daily.locations,
      daily.locations.map((r) => r.visits),
      seedFor('locations'),
    ),
    sources: buildShuffledPool(
      daily.sources,
      daily.sources.map((r) => r.visits),
      seedFor('sources'),
    ),
    pages: buildShuffledPool(
      daily.pages,
      daily.pages.map((r) => r.pageviews),
      seedFor('pages'),
    ),
    entryPages: buildShuffledPool(
      daily.entryPages,
      daily.entryPages.map((r) => r.entrances),
      seedFor('entryPages'),
    ),
    exitPages: buildShuffledPool(
      daily.exitPages,
      daily.exitPages.map((r) => r.exits),
      seedFor('exitPages'),
    ),
  }
}

function formatDateTime(epochSec: number): string {
  const d = new Date(epochSec * 1000)
  const yyyy = d.getUTCFullYear()
  const mm = String(d.getUTCMonth() + 1).padStart(2, '0')
  const dd = String(d.getUTCDate()).padStart(2, '0')
  const hh = String(d.getUTCHours()).padStart(2, '0')
  const mi = String(d.getUTCMinutes()).padStart(2, '0')
  const ss = String(d.getUTCSeconds()).padStart(2, '0')
  return `${yyyy}-${mm}-${dd} ${hh}:${mi}:${ss}`
}

function dateToEpochSec(date: string): number {
  return Math.floor(new Date(`${date}T00:00:00Z`).getTime() / 1000)
}

export class PlausibleMapper implements ImportMapper {
  readonly provider = 'plausible'
  readonly expectedFileExtension = '.zip'

  private async extractCsvs(
    zipPath: string,
    tempDir: string,
  ): Promise<Map<string, string[]>> {
    const stat = await fs.promises.stat(zipPath)
    if (stat.size > MAX_PLAUSIBLE_ZIP_BYTES) {
      throw new ImportError('Plausible ZIP exceeds the 100 MB upload limit.')
    }

    const found = new Map<string, string[]>()
    const openFiles = new Set<number>()
    let totalUncompressed = 0
    let entryCount = 0
    let pendingEntries = 0
    const unzipper = new Unzip((file) => {
      if (++entryCount > 1000) {
        throw new ImportError('Plausible ZIP contains too many files.')
      }
      const key = Object.keys(FILE_PATTERNS).find((key) =>
        FILE_PATTERNS[key].test(file.name),
      )
      if (file.originalSize > MAX_PLAUSIBLE_UNCOMPRESSED_BYTES) {
        throw new ImportError(
          'Plausible ZIP exceeds the 2 GiB uncompressed limit. Please export a shorter date range.',
        )
      }
      let fd: number | undefined
      if (key) {
        const csvPath = path.join(tempDir, `${entryCount}.csv`)
        fd = fs.openSync(csvPath, 'wx')
        openFiles.add(fd)
        found.set(key, [...(found.get(key) ?? []), csvPath])
      }
      pendingEntries++
      file.ondata = (error, chunk, final) => {
        if (error) throw error
        totalUncompressed += chunk.length
        if (totalUncompressed > MAX_PLAUSIBLE_UNCOMPRESSED_BYTES) {
          throw new ImportError(
            'Plausible ZIP exceeds the 2 GiB uncompressed limit. Please export a shorter date range.',
          )
        }
        if (fd !== undefined) {
          let offset = 0
          while (offset < chunk.length) {
            offset += fs.writeSync(fd, chunk, offset, chunk.length - offset)
          }
          if (final) {
            fs.closeSync(fd)
            openFiles.delete(fd)
            fd = undefined
          }
        }
        if (final) pendingEntries--
      }
      file.start()
    })
    unzipper.register(UnzipInflate)

    try {
      for await (const chunk of fs.createReadStream(zipPath, {
        highWaterMark: 16 * 1024,
      })) {
        unzipper.push(chunk as Buffer)
      }
      unzipper.push(new Uint8Array(0), true)
      if (pendingEntries > 0) throw new Error('Incomplete ZIP entry')
    } catch (error) {
      if (error instanceof ImportError) throw error
      throw new ImportError(
        'Could not read the uploaded ZIP archive. Please upload the full export ZIP from Plausible again.',
      )
    } finally {
      for (const fd of openFiles) fs.closeSync(fd)
    }
    if (!found.has('visitors')) {
      throw new ImportError(
        'ZIP does not contain imported_visitors_*.csv. Please upload the full export ZIP from Plausible.',
      )
    }
    return found
  }

  private async partitionCsvs(csvs: Map<string, string[]>, tempDir: string) {
    const days = new Map<string, Map<string, string[]>>()
    const daySizes = new Map<string, number>()
    const buffers = new Map<string, string[]>()
    let bufferedBytes = 0
    const flush = async () => {
      for (const [filePath, rows] of buffers) {
        await fs.promises.appendFile(filePath, rows.join(''))
      }
      buffers.clear()
      bufferedBytes = 0
    }
    const required: Record<string, string[]> = {
      visitors: ['date', 'visits', 'pageviews'],
      pages: ['date', 'page', 'pageviews'],
      browsers: ['date', 'browser', 'visits'],
      devices: ['date', 'device', 'visits'],
      operatingSystems: ['date', 'operating_system', 'visits'],
      locations: ['date', 'country', 'visits'],
      sources: ['date', 'source', 'visits'],
      entryPages: ['date', 'entry_page', 'entrances'],
      exitPages: ['date', 'exit_page', 'exits'],
      customEvents: ['date', 'name', 'events'],
    }
    for (const [key, paths] of csvs) {
      for (const csvPath of paths) {
        for await (const row of readCsvRows(csvPath, {
          columns: (headers: string[]) => {
            const missing = required[key].filter(
              (column) => !headers.includes(column),
            )
            if (missing.length)
              throw new ImportError(
                `Plausible ${key} CSV is missing columns: ${missing.join(', ')}.`,
              )
            return headers
          },
        })) {
          const date = normalizeNull(row.date)
          if (!date || !DATE_REGEX.test(date)) continue
          const parsedDate = new Date(`${date}T00:00:00Z`)
          if (
            !Number.isFinite(parsedDate.getTime()) ||
            parsedDate.toISOString().slice(0, 10) !== date
          )
            continue
          const line = `${JSON.stringify(row)}\n`
          const bytes = Buffer.byteLength(line)
          const daySize = (daySizes.get(date) ?? 0) + bytes
          if (daySize > MAX_DAILY_BYTES)
            throw new ImportError(
              `Plausible data for ${date} exceeds the 128 MiB daily processing limit. Please contact support.`,
            )
          daySizes.set(date, daySize)
          const dayFiles = days.get(date) ?? new Map<string, string[]>()
          const filePath = path.join(tempDir, `${date}-${key}.jsonl`)
          dayFiles.set(key, [filePath])
          days.set(date, dayFiles)
          const rows = buffers.get(filePath) ?? []
          rows.push(line)
          buffers.set(filePath, rows)
          bufferedBytes += bytes
          if (bufferedBytes >= 1024 * 1024) await flush()
        }
        await fs.promises.unlink(csvPath)
      }
    }
    await flush()
    return days
  }

  private async buildDailyData(
    csvs: Map<string, string[]>,
  ): Promise<Map<string, DailyData>> {
    const visitorsFiles = csvs.get('visitors')
    if (!visitorsFiles) {
      throw new ImportError(
        'ZIP does not contain imported_visitors_*.csv. Please upload the full export ZIP from Plausible.',
      )
    }

    const days = new Map<string, DailyData>()

    const ensureDay = (date: string): DailyData => {
      let d = days.get(date)
      if (!d) {
        d = {
          visitors: {
            date,
            visitors: 0,
            pageviews: 0,
            visits: 0,
            bounces: 0,
            visit_duration: 0,
          },
          browsers: [],
          devices: [],
          os: [],
          locations: [],
          sources: [],
          pages: [],
          entryPages: [],
          exitPages: [],
          customEvents: [],
        }
        days.set(date, d)
      }
      return d
    }

    for await (const row of readDailyRows(visitorsFiles)) {
      const date = normalizeNull(row.date)
      if (!date || !DATE_REGEX.test(date)) continue
      const visits = toInt(row.visits, 'visits')
      const pageviews = toInt(row.pageviews, 'pageviews')
      const day = ensureDay(date)
      day.visitors = {
        date,
        visitors: day.visitors.visitors + toInt(row.visitors, 'visitors'),
        pageviews: day.visitors.pageviews + pageviews,
        visits: day.visitors.visits + visits,
        bounces: day.visitors.bounces + toInt(row.bounces, 'bounces'),
        visit_duration:
          day.visitors.visit_duration +
          toInt(row.visit_duration, 'visit_duration'),
      }
    }

    if (days.size === 0) {
      throw new ImportError(
        'imported_visitors_*.csv contains no valid daily rows.',
      )
    }

    const pagesFiles = csvs.get('pages')
    if (pagesFiles) {
      for await (const row of readDailyRows(pagesFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const pageviews = toInt(row.pageviews)
        if (pageviews <= 0) continue
        days.get(date)!.pages.push({
          hostname: normalizeNull(row.hostname),
          page: normalizeNull(row.page),
          pageviews,
        })
      }
    }

    const browsersFiles = csvs.get('browsers')
    if (browsersFiles) {
      for await (const row of readDailyRows(browsersFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const visits = toInt(row.visits)
        if (visits <= 0) continue
        days.get(date)!.browsers.push({
          browser: normalizeNull(row.browser),
          browser_version: normalizeNull(row.browser_version),
          visits,
        })
      }
    }

    const devicesFiles = csvs.get('devices')
    if (devicesFiles) {
      for await (const row of readDailyRows(devicesFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const visits = toInt(row.visits)
        if (visits <= 0) continue
        days.get(date)!.devices.push({
          device: normalizeNull(row.device),
          visits,
        })
      }
    }

    const osFiles = csvs.get('operatingSystems')
    if (osFiles) {
      for await (const row of readDailyRows(osFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const visits = toInt(row.visits)
        if (visits <= 0) continue
        days.get(date)!.os.push({
          operating_system: normalizeNull(row.operating_system),
          operating_system_version: normalizeNull(row.operating_system_version),
          visits,
        })
      }
    }

    const locationsFiles = csvs.get('locations')
    if (locationsFiles) {
      for await (const row of readDailyRows(locationsFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const visits = toInt(row.visits)
        if (visits <= 0) continue
        days.get(date)!.locations.push({
          country: normalizeNull(row.country),
          region: normalizeNull(row.region),
          visits,
        })
      }
    }

    const sourcesFiles = csvs.get('sources')
    if (sourcesFiles) {
      for await (const row of readDailyRows(sourcesFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const visits = toInt(row.visits)
        if (visits <= 0) continue
        days.get(date)!.sources.push({
          source: normalizeNull(row.source),
          referrer: normalizeNull(row.referrer),
          utm_source: normalizeNull(row.utm_source),
          utm_medium: normalizeNull(row.utm_medium),
          utm_campaign: normalizeNull(row.utm_campaign),
          utm_content: normalizeNull(row.utm_content),
          utm_term: normalizeNull(row.utm_term),
          visits,
        })
      }
    }

    const entryPagesFiles = csvs.get('entryPages')
    if (entryPagesFiles) {
      for await (const row of readDailyRows(entryPagesFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const entrances = toInt(row.entrances)
        if (entrances <= 0) continue
        days.get(date)!.entryPages.push({
          entry_page: normalizeNull(row.entry_page),
          entrances,
        })
      }
    }

    const exitPagesFiles = csvs.get('exitPages')
    if (exitPagesFiles) {
      for await (const row of readDailyRows(exitPagesFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const exits = toInt(row.exits)
        if (exits <= 0) continue
        days.get(date)!.exitPages.push({
          exit_page: normalizeNull(row.exit_page),
          exits,
        })
      }
    }

    const customEventsFiles = csvs.get('customEvents')
    if (customEventsFiles) {
      for await (const row of readDailyRows(customEventsFiles)) {
        const date = normalizeNull(row.date)
        if (!date || !days.has(date)) continue
        const name = normalizeNull(row.name)
        const events = toInt(row.events, 'events')
        if (!name || events <= 0) continue
        days
          .get(date)!
          .customEvents.push({ name, events, url: normalizeNull(row.link_url) })
      }
    }

    return days
  }

  async *createRowStream(
    filePath: string,
    pid: string,
    importID: number,
  ): AsyncIterable<AnalyticsImportRow> {
    const tempDir = await fs.promises.mkdtemp(
      path.join(os.tmpdir(), 'swetrix-plausible-'),
    )
    try {
      const csvs = await this.extractCsvs(filePath, tempDir)
      const partitions = await this.partitionCsvs(csvs, tempDir)
      let yielded = 0
      let processedDays = 0
      for (const date of Array.from(partitions.keys()).sort()) {
        const files = partitions.get(date)!
        if (!files.has('visitors')) continue
        const days = await this.buildDailyData(files)
        for (const daily of days.values()) {
          processedDays++
          const events = daily.customEvents.reduce(
            (total, event) => total + event.events,
            0,
          )
          if (
            daily.visitors.visits > MAX_SESSIONS_PER_DAY ||
            events > MAX_EVENTS_PER_DAY ||
            daily.visitors.pageviews >
              Math.max(daily.visitors.visits, 1) * MAX_PAGEVIEWS_PER_SESSION
          ) {
            throw new ImportError(
              `Plausible data for ${date} exceeds the daily event processing limits. Please contact support.`,
            )
          }
          if (
            yielded + daily.visitors.pageviews + events >
            MAX_GENERATED_ROWS_PER_IMPORT
          ) {
            throw new ImportError(
              `Plausible export would generate more than ${MAX_GENERATED_ROWS_PER_IMPORT} rows. Please export a shorter date range.`,
            )
          }
          for (const row of this.synthesizeDay(daily, pid, importID)) {
            yielded++
            if (yielded % 5000 === 0)
              await new Promise<void>((resolve) => setImmediate(resolve))
            yield row
          }
        }
        for (const paths of files.values()) {
          for (const filePath of paths) await fs.promises.unlink(filePath)
        }
      }
      if (!processedDays)
        throw new ImportError('Plausible export contains no valid daily rows.')
    } finally {
      await fs.promises.rm(tempDir, { recursive: true, force: true })
    }
  }

  private *synthesizeDay(
    daily: DailyData,
    pid: string,
    importID: number,
  ): Iterable<AnalyticsImportRow> {
    const { date } = daily.visitors
    const visits = Math.min(
      Math.max(daily.visitors.visits, 0),
      MAX_SESSIONS_PER_DAY,
    )
    const pageviews = Math.max(daily.visitors.pageviews, 0)

    if (pageviews === 0 && daily.customEvents.length === 0) return

    const sessionCount =
      visits > 0
        ? visits
        : pageviews > 0 || daily.customEvents.length > 0
          ? 1
          : 0
    if (sessionCount === 0) return

    const pools = buildPools(daily, importID, date)
    const fallbackHost = pickSiteHostname(daily.pages)
    const dayStartSec = dateToEpochSec(date)
    const dayEndSec = dayStartSec + 86399
    const sessionSpacingSec =
      sessionCount > 1 ? Math.floor(86400 / sessionCount) : 86400

    const cappedPageviews = Math.min(
      pageviews,
      sessionCount * MAX_PAGEVIEWS_PER_SESSION,
    )
    const basePerSession = Math.min(
      Math.floor(cappedPageviews / sessionCount),
      MAX_PAGEVIEWS_PER_SESSION,
    )
    const remainder = cappedPageviews % sessionCount

    const avgVisitDuration =
      sessionCount > 0 ? daily.visitors.visit_duration / sessionCount : 0

    interface SessionInfo {
      psid: string
      browser: BrowserRow | null
      device: DeviceRow | null
      os: OsRow | null
      location: LocationRow | null
      source: SourceRow | null
      host: string | null
      startSec: number
    }

    const sessions: SessionInfo[] = []

    for (let i = 0; i < sessionCount; i++) {
      const psid = sessionIdToPsid(`${importID}|${date}|${i}`)

      const browser = pickFromPool(pools.browsers, i)
      const device = pickFromPool(pools.devices, i)
      const os = pickFromPool(pools.os, i)
      const location = pickFromPool(pools.locations, i)
      const source = pickFromPool(pools.sources, i)

      const startSec = dayStartSec + Math.floor((i * 86400) / sessionCount)

      sessions.push({
        psid,
        browser,
        device,
        os,
        location,
        source,
        host: fallbackHost,
        startSec,
      })
    }

    let middlePageIdx = 0

    for (let i = 0; i < sessions.length; i++) {
      const session = sessions[i]
      const pvCount = basePerSession + (i < remainder ? 1 : 0)
      if (pvCount <= 0) continue

      const entryPage = pickFromPool(pools.entryPages, i)
      const exitPage = pickFromPool(pools.exitPages, i)

      const pages: PageRow[] = []
      const entryPagePath = entryPage?.entry_page ?? null
      const exitPagePath = exitPage?.exit_page ?? null

      for (let j = 0; j < pvCount; j++) {
        if (j === 0 && entryPagePath) {
          pages.push({
            hostname: session.host,
            page: entryPagePath,
            pageviews: 1,
          })
          continue
        }
        if (j === pvCount - 1 && pvCount > 1 && exitPagePath) {
          pages.push({
            hostname: session.host,
            page: exitPagePath,
            pageviews: 1,
          })
          continue
        }
        const sampled = pickFromPool(pools.pages, middlePageIdx++)
        if (sampled) {
          pages.push(sampled)
        } else if (entryPagePath || exitPagePath) {
          pages.push({
            hostname: session.host,
            page: entryPagePath ?? exitPagePath,
            pageviews: 1,
          })
        } else {
          pages.push({ hostname: session.host, page: '/', pageviews: 1 })
        }
      }

      let pageGapSec = MIN_PAGE_GAP_SEC
      if (pvCount > 1) {
        pageGapSec = Math.round(avgVisitDuration / Math.max(pvCount - 1, 1))
        if (pageGapSec < MIN_PAGE_GAP_SEC) pageGapSec = MIN_PAGE_GAP_SEC
        if (pageGapSec > MAX_PAGE_GAP_SEC) pageGapSec = MAX_PAGE_GAP_SEC
      }

      for (let j = 0; j < pages.length; j++) {
        const page = pages[j]
        const rawTsSec = session.startSec + j * pageGapSec
        const tsSec = rawTsSec > dayEndSec ? dayEndSec : rawTsSec
        const created = formatDateTime(tsSec)

        const data = this.buildEventData({
          pid,
          importID,
          psid: session.psid,
          browser: session.browser,
          device: session.device,
          os: session.os,
          location: session.location,
          source: session.source,
          host: page.hostname ?? session.host,
          page: page.page,
          created,
        })

        yield { type: 'pageview', data }
      }
    }

    if (daily.customEvents.length > 0 && sessions.length > 0) {
      for (const ev of daily.customEvents) {
        const eventSeed = seedFromString(`${importID}|${date}|ev|${ev.name}`)
        const evPrng = seededPrng(eventSeed)
        const eventCount = Math.min(Math.max(ev.events, 0), MAX_EVENTS_PER_DAY)

        for (let k = 0; k < eventCount; k++) {
          const sessionIdx = Math.floor(evPrng() * sessions.length)
          const session = sessions[Math.min(sessionIdx, sessions.length - 1)]
          const tsSec =
            session.startSec + Math.floor(evPrng() * sessionSpacingSec)
          const created = formatDateTime(tsSec)

          const data = this.buildEventData({
            pid,
            importID,
            psid: session.psid,
            browser: session.browser,
            device: session.device,
            os: session.os,
            location: session.location,
            source: session.source,
            host: session.host,
            page: null,
            created,
          })
          if (ev.name === 'Outbound Link: Click') {
            data.outbound_url = truncate(ev.url, 2048)
            yield { type: 'outbound', data }
          } else {
            data.event_name = truncate(ev.name, 256) || ''
            yield { type: 'custom_event', data }
          }
        }
      }
    }
  }

  private buildEventData(args: {
    pid: string
    importID: number
    psid: string
    browser: BrowserRow | null
    device: DeviceRow | null
    os: OsRow | null
    location: LocationRow | null
    source: SourceRow | null
    host: string | null
    page: string | null
    created: string
  }): Record<string, unknown> {
    const device = mapDevice(args.device?.device ?? null)
    const browser = mapBrowser(args.browser?.browser ?? null, device)
    const os = mapOS(args.os?.operating_system ?? null)

    const cc = args.location?.country
    const validCC = cc && /^[A-Z]{2}$/.test(cc) ? cc : null
    const rgc = parseRegion(args.location?.region ?? null)

    const ref = buildReferrer(
      args.source?.referrer ?? null,
      args.source?.source ?? null,
      args.host,
    )

    return {
      psid: args.psid,
      profileId: null,
      pid: args.pid,
      host: truncate(args.host, 253),
      pg: truncate(args.page, 2048),
      dv: device,
      br: truncate(browser, 30),
      brv: truncate(args.browser?.browser_version ?? null, 20),
      os: truncate(os, 25),
      osv: truncate(args.os?.operating_system_version ?? null, 20),
      lc: null,
      ref: truncate(ref, 2048),
      so: truncate(args.source?.utm_source ?? null, 256),
      me: truncate(args.source?.utm_medium ?? null, 256),
      ca: truncate(args.source?.utm_campaign ?? null, 256),
      te: truncate(args.source?.utm_term ?? null, 256),
      co: truncate(args.source?.utm_content ?? null, 256),
      cc: validCC,
      rg: null,
      rgc,
      ct: null,
      'meta.key': [],
      'meta.value': [],
      importID: args.importID,
      created: args.created,
    }
  }
}