Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

10.7 KB
Raw Blame
import * as fs from 'fs'
import * as path from 'path'
import * as os from 'os'
import { Unzip, UnzipInflate } from 'fflate'
import { randomUUID } from 'crypto'
import { readCsvRows } from './csv.utils'

import {
  ImportMapper,
  ImportError,
  AnalyticsImportRow,
} from './mapper.interface'
import {
  normalizeNull,
  truncate,
  sessionIdToPsid,
  MOBILE_BROWSER_VARIANTS,
} from './mapper.utils'

const WEBSITE_EVENT_CSV = 'website_event.csv'
const MAX_UMAMI_CSV_BYTES = 512 * 1024 * 1024

const BROWSER_MAP: Record<string, string> = {
  chrome: 'Chrome',
  opera: 'Opera',
  crios: 'Mobile Chrome',
  firefox: 'Firefox',
  facebook: 'Facebook',
  safari: 'Safari',
  ios: 'Mobile Safari',
  'ios-webview': 'Mobile Safari',
  'edge-chromium': 'Edge',
  samsung: 'Samsung Internet',
  yandexbrowser: 'Yandex',
  'edge-ios': 'Edge',
  'chromium-webview': 'Chrome WebView',
  fxios: 'Mobile Firefox',
  edge: 'Edge',
  ie: 'Internet Explorer',
  'internet explorer': 'Internet Explorer',
}

const OS_MAP: Record<string, string> = {
  'mac os': 'macOS',
  'windows 10': 'Windows',
  'windows 7': 'Windows',
  'windows 8': 'Windows',
  'windows 8.1': 'Windows',
  'windows vista': 'Windows',
  'windows xp': 'Windows',
  'windows server 2003': 'Windows',
  'android os': 'Android',
  'chrome os': 'Chrome OS',
  'chromium os': 'Chrome OS',
}

const DEVICE_MAP: Record<string, string> = {
  laptop: 'desktop',
  desktop: 'desktop',
  mobile: 'mobile',
  tablet: 'tablet',
}

function mapBrowser(raw: string | null, device: string | null): string | null {
  if (!raw) return null

  const key = raw.toLowerCase()
  let mapped = BROWSER_MAP[key]

  if (!mapped) {
    mapped = raw.charAt(0).toUpperCase() + raw.slice(1)
  }

  const isMobile = device === 'mobile' || device === 'tablet'
  if (isMobile && MOBILE_BROWSER_VARIANTS[mapped]) {
    return MOBILE_BROWSER_VARIANTS[mapped]
  }

  return mapped
}

function mapOS(raw: string | null): {
  name: string | null
  version: string | null
} {
  if (!raw) return { name: null, version: null }

  const windowsMatch = raw.match(/^windows(?:\s+(.+))?$/i)
  const windowsVersion = windowsMatch?.[1]?.trim() || null
  const windowsVersionKey = windowsVersion?.toLowerCase()

  if (
    windowsMatch &&
    windowsVersionKey !== 'phone' &&
    windowsVersionKey !== 'mobile'
  ) {
    return { name: 'Windows', version: windowsVersion }
  }

  const key = raw.toLowerCase()
  return { name: OS_MAP[key] ?? raw, version: null }
}

function mapDevice(raw: string | null): string {
  if (!raw) return 'desktop'
  const key = raw.toLowerCase()
  return DEVICE_MAP[key] ?? 'desktop'
}

function buildReferrer(
  domain: string | null,
  path: string | null,
  query: string | null,
  hostname: string | null,
): string | null {
  if (!domain) return null

  let ref = `https://${domain}`
  if (path) ref += path
  if (query) ref += query.startsWith('?') ? query : `?${query}`

  if (hostname) {
    const cleanHost = hostname.replace(/^www\./, '')
    const cleanDomain = domain.replace(/^www\./, '')
    if (cleanDomain === cleanHost) return null
  }

  return ref
}

const DATE_REGEX =
  /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/

function isValidTimestamp(ts: string): boolean {
  return DATE_REGEX.test(ts)
}

export class UmamiMapper implements ImportMapper {
  readonly provider = 'umami'
  readonly expectedFileExtension = '.zip'

  private async extractCsvFromZip(zipPath: string): Promise<string> {
    const tempCsv = path.join(os.tmpdir(), `swetrix-umami-${randomUUID()}.csv`)

    return new Promise((resolve, reject) => {
      const zipStream = fs.createReadStream(zipPath)
      const unzipper = new Unzip()

      unzipper.register(UnzipInflate)

      let settled = false
      let foundEntry = false
      let fileDescriptor: number | null = null
      let tempFileCreated = false
      let extractedBytes = 0
      let terminateEntry: (() => void) | null = null

      const cleanupTempCsv = () => {
        if (fileDescriptor !== null) {
          try {
            fs.closeSync(fileDescriptor)
          } catch {
            //
          }
          fileDescriptor = null
        }

        if (tempFileCreated) {
          try {
            fs.unlinkSync(tempCsv)
          } catch {
            //
          }
        }
      }

      const fail = (error: Error) => {
        if (settled) return
        settled = true
        terminateEntry?.()
        zipStream.destroy()
        cleanupTempCsv()
        reject(error)
      }

      const succeed = () => {
        if (settled) return
        settled = true
        if (fileDescriptor !== null) {
          try {
            fs.closeSync(fileDescriptor)
          } catch (error) {
            cleanupTempCsv()
            reject(error)
            return
          }
          fileDescriptor = null
        }
        zipStream.destroy()
        resolve(tempCsv)
      }

      unzipper.onfile = (file) => {
        if (path.basename(file.name) !== WEBSITE_EVENT_CSV || foundEntry) {
          let ignoredBytes = 0
          file.ondata = (error, chunk) => {
            if (error) {
              fail(error)
              return
            }
            ignoredBytes += chunk.length
            if (ignoredBytes > MAX_UMAMI_CSV_BYTES) {
              fail(
                new ImportError(
                  'Umami ZIP contains an entry above the 512 MiB decompression limit.',
                ),
              )
            }
          }
          file.start()
          return
        }

        foundEntry = true

        if (
          typeof file.originalSize === 'number' &&
          Number.isFinite(file.originalSize) &&
          file.originalSize > MAX_UMAMI_CSV_BYTES
        ) {
          fail(
            new ImportError(
              `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
            ),
          )
          return
        }

        try {
          fileDescriptor = fs.openSync(tempCsv, 'wx')
          tempFileCreated = true
        } catch (error) {
          fail(error as Error)
          return
        }

        terminateEntry = file.terminate
        file.ondata = (error, chunk, final) => {
          if (error) {
            fail(error)
            return
          }

          extractedBytes += chunk.length
          if (extractedBytes > MAX_UMAMI_CSV_BYTES) {
            fail(
              new ImportError(
                `${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
              ),
            )
            return
          }

          try {
            fs.writeSync(fileDescriptor!, chunk)
          } catch (writeError) {
            fail(writeError as Error)
            return
          }

          if (final) {
            succeed()
          }
        }

        try {
          file.start()
        } catch (error) {
          fail(error as Error)
        }
      }

      zipStream.on('data', (chunk: Buffer) => {
        if (settled) return

        try {
          unzipper.push(
            new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength),
          )
        } catch (error) {
          fail(error as Error)
        }
      })

      zipStream.on('end', () => {
        if (settled) return

        try {
          unzipper.push(new Uint8Array(0), true)
        } catch (error) {
          fail(error as Error)
          return
        }

        if (!foundEntry) {
          fail(
            new ImportError(
              `ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
            ),
          )
        }
      })

      zipStream.on('error', (error) => {
        fail(error)
      })
    })
  }

  async *createRowStream(
    filePath: string,
    pid: string,
    importID: number,
  ): AsyncIterable<AnalyticsImportRow> {
    const csvPath = await this.extractCsvFromZip(filePath)

    let cleanedUp = false
    const cleanup = () => {
      if (!cleanedUp) {
        cleanedUp = true
        try {
          fs.unlinkSync(csvPath)
        } catch {
          //
        }
      }
    }

    try {
      const parser = readCsvRows(csvPath, {
        columns: true,
        skip_empty_lines: true,
        trim: true,
        relax_quotes: true,
        relax_column_count: true,
      })

      for await (const row of parser) {
        const sessionId = normalizeNull(row.session_id)
        if (!sessionId) continue

        const createdAt = normalizeNull(row.created_at)
        if (!createdAt || !isValidTimestamp(createdAt)) continue

        const eventType = normalizeNull(row.event_type)
        if (!eventType || (eventType !== '1' && eventType !== '2')) continue

        const psid = sessionIdToPsid(sessionId)

        const rawDevice = normalizeNull(row.device)
        const device = mapDevice(rawDevice)
        const browser = mapBrowser(normalizeNull(row.browser), rawDevice)
        const os = mapOS(normalizeNull(row.os))

        const hostname = truncate(normalizeNull(row.hostname), 253)
        const referrerDomain = normalizeNull(row.referrer_domain)
        const referrerPath = normalizeNull(row.referrer_path)
        const referrerQuery = normalizeNull(row.referrer_query)
        const ref = buildReferrer(
          referrerDomain,
          referrerPath,
          referrerQuery,
          hostname,
        )

        const country = normalizeNull(row.country)
        const cc = country && /^[A-Z]{2}$/.test(country) ? country : null

        const region = normalizeNull(row.region)

        const baseData: Record<string, unknown> = {
          psid,
          profileId: null,
          pid,
          host: hostname,
          pg: truncate(normalizeNull(row.url_path), 2048),
          dv: device,
          br: truncate(browser, 30),
          brv: null,
          os: truncate(os.name, 25),
          osv: truncate(os.version, 20),
          lc: truncate(normalizeNull(row.language), 35),
          ref: truncate(ref, 2048),
          so: truncate(normalizeNull(row.utm_source), 256),
          me: truncate(normalizeNull(row.utm_medium), 256),
          ca: truncate(normalizeNull(row.utm_campaign), 256),
          te: truncate(normalizeNull(row.utm_term), 256),
          co: truncate(normalizeNull(row.utm_content), 256),
          cc,
          rg: null,
          rgc: region && region.length <= 10 ? region : null,
          ct: truncate(normalizeNull(row.city), 60),
          'meta.key': [],
          'meta.value': [],
          importID,
          created: createdAt,
        }

        if (eventType === '1') {
          yield { type: 'pageview', data: baseData }
        } else {
          const eventName = truncate(normalizeNull(row.event_name), 256) || ''
          yield {
            type: 'custom_event',
            data: { ...baseData, event_name: eventName },
          }
        }
      }
    } finally {
      cleanup()
    }
  }
}