Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

13.7 KB
Raw Blame
import { BetaAnalyticsDataClient } from '@google-analytics/data'
import { OAuth2Client } from 'google-auth-library'
import * as crypto from 'crypto'
import dayjs from 'dayjs'

import {
  ImportMapper,
  ImportError,
  AnalyticsImportRow,
} from './mapper.interface'
import {
  truncate,
  sessionIdToPsid,
  MOBILE_BROWSER_VARIANTS,
} from './mapper.utils'
import { deriveKey } from '../../common/utils'

const ENCRYPTION_KEY = Buffer.from(deriveKey('ga4-token', 32), 'hex')

const PAGE_SIZE = 100_000

const EXCLUDED_EVENTS = new Set([
  'page_view',
  'session_start',
  'first_visit',
  'scroll',
  'click',
  'file_download',
  'form_start',
  'form_submit',
  'view_search_results',
  'video_start',
  'video_progress',
  'video_complete',
])

const DEVICE_MAP: Record<string, string> = {
  desktop: 'desktop',
  mobile: 'mobile',
  tablet: 'tablet',
}

function mapDevice(raw: string | null): string {
  if (!raw) return 'desktop'
  return DEVICE_MAP[raw.toLowerCase()] ?? 'desktop'
}

function mapBrowser(raw: string | null, device: string): string | null {
  if (!raw || raw === '(not set)') return null
  const isMobile = device === 'mobile' || device === 'tablet'
  if (isMobile && MOBILE_BROWSER_VARIANTS[raw]) {
    return MOBILE_BROWSER_VARIANTS[raw]
  }
  return raw
}

function mapOS(raw: string | null): string | null {
  if (!raw || raw === '(not set)') return null
  if (raw === 'Macintosh') return 'macOS'
  return raw
}

function normalizeGA4Value(val: string | null | undefined): string | null {
  if (!val || val === '(not set)' || val === '(none)' || val === '(direct)') {
    return null
  }
  return val
}

function buildReferrer(source: string | null): string | null {
  if (!source) return null
  if (source.includes('://')) return source
  if (source.includes('.')) return `https://${source}`
  return null
}

function decryptToken(ciphertext: string): string {
  const parts = ciphertext.split(':')
  if (parts.length !== 3) {
    throw new Error('Invalid encrypted token format')
  }
  const [ivHex, encHex, tagHex] = parts
  const decipher = crypto.createDecipheriv(
    'aes-256-gcm',
    ENCRYPTION_KEY,
    Buffer.from(ivHex, 'hex'),
  )
  decipher.setAuthTag(Buffer.from(tagHex, 'hex'))
  const decrypted = Buffer.concat([
    decipher.update(Buffer.from(encHex, 'hex')),
    decipher.final(),
  ])
  return decrypted.toString('utf8')
}

function getMonthChunks(
  startDate: string,
  endDate: string,
): { start: string; end: string }[] {
  const chunks: { start: string; end: string }[] = []
  let current = dayjs(startDate)
  const end = dayjs(endDate)

  while (current.isBefore(end) || current.isSame(end, 'day')) {
    const monthEnd = current.endOf('month')
    const chunkEnd = monthEnd.isAfter(end) ? end : monthEnd
    chunks.push({
      start: current.format('YYYY-MM-DD'),
      end: chunkEnd.format('YYYY-MM-DD'),
    })
    current = chunkEnd.add(1, 'day')
  }

  return chunks
}

interface SourceInfo {
  so: string | null
  me: string | null
  ca: string | null
  te: string | null
  co: string | null
  ref: string | null
}

export class Ga4Mapper implements ImportMapper {
  readonly provider = 'google-analytics'
  readonly expectedFileExtension = null

  async *createRowStream(
    _filePath: string | null,
    pid: string,
    importID: number,
    context?: Record<string, unknown>,
  ): AsyncIterable<AnalyticsImportRow> {
    if (
      !context?.encryptedRefreshToken ||
      !context?.ga4PropertyId ||
      !context?.ga4ClientId ||
      !context?.ga4ClientSecret
    ) {
      throw new ImportError(
        'GA4 import requires OAuth credentials and a property ID.',
      )
    }

    const refreshToken = decryptToken(context.encryptedRefreshToken as string)
    const propertyId = context.ga4PropertyId as string
    const clientId = context.ga4ClientId as string
    const clientSecret = context.ga4ClientSecret as string

    const oauth2Client = new OAuth2Client(clientId, clientSecret)
    oauth2Client.setCredentials({ refresh_token: refreshToken })

    const client = new BetaAnalyticsDataClient({
      authClient: oauth2Client as any,
    })
    const property = propertyId.startsWith('properties/')
      ? propertyId
      : `properties/${propertyId}`

    const startDate = (context.startDate as string) || '2020-01-01'
    const endDate = (context.endDate as string) || dayjs().format('YYYY-MM-DD')
    const chunks = getMonthChunks(startDate, endDate)

    for (const chunk of chunks) {
      const sourceLookup = await this.buildSourceLookup(client, property, chunk)
      yield* this.fetchPageviews(
        client,
        property,
        chunk,
        pid,
        importID,
        sourceLookup,
      )
      yield* this.fetchCustomEvents(client, property, chunk, pid, importID)
    }
  }

  /**
   * Fetches source/UTM data into a lookup map keyed by (dateHour|hostName|pagePath).
   * Kept in memory per monthly chunk -- the map stores only the highest-traffic
   * source combo for each key, so memory stays bounded.
   */
  private async buildSourceLookup(
    client: BetaAnalyticsDataClient,
    property: string,
    dateRange: { start: string; end: string },
  ): Promise<Map<string, SourceInfo>> {
    const lookup = new Map<string, SourceInfo & { count: number }>()
    let offset = 0

    while (true) {
      const [report] = await client.runReport({
        property,
        dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
        dimensions: [
          { name: 'dateHour' },
          { name: 'hostName' },
          { name: 'pagePath' },
          { name: 'sessionSource' },
          { name: 'sessionMedium' },
          { name: 'sessionCampaignName' },
          { name: 'sessionManualTerm' },
          { name: 'sessionManualAdContent' },
        ],
        metrics: [{ name: 'screenPageViews' }],
        limit: PAGE_SIZE,
        offset,
        keepEmptyRows: false,
      })

      const rows = report.rows || []
      if (rows.length === 0) break

      for (const row of rows) {
        const dims = row.dimensionValues || []
        const metrics = row.metricValues || []

        const dateHour = dims[0]?.value || ''
        const host = normalizeGA4Value(dims[1]?.value)
        const pagePath = normalizeGA4Value(dims[2]?.value)
        const source = normalizeGA4Value(dims[3]?.value)
        const medium = normalizeGA4Value(dims[4]?.value)
        const campaign = normalizeGA4Value(dims[5]?.value)
        const term = normalizeGA4Value(dims[6]?.value)
        const content = normalizeGA4Value(dims[7]?.value)

        const count = parseInt(metrics[0]?.value || '0', 10)
        if (count <= 0) continue

        const key = `${dateHour}|${host || ''}|${pagePath || ''}`
        const existing = lookup.get(key)

        if (!existing || count > existing.count) {
          lookup.set(key, {
            so: source,
            me: medium,
            ca: campaign,
            te: term,
            co: content,
            ref: buildReferrer(source),
            count,
          })
        }
      }

      if (rows.length < PAGE_SIZE) break
      offset += PAGE_SIZE
    }

    return lookup
  }

  private async *fetchPageviews(
    client: BetaAnalyticsDataClient,
    property: string,
    dateRange: { start: string; end: string },
    pid: string,
    importID: number,
    sourceLookup: Map<string, SourceInfo>,
  ): AsyncIterable<AnalyticsImportRow> {
    let offset = 0

    while (true) {
      const [report] = await client.runReport({
        property,
        dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
        dimensions: [
          { name: 'dateHour' },
          { name: 'hostName' },
          { name: 'pagePath' },
          { name: 'deviceCategory' },
          { name: 'browser' },
          { name: 'operatingSystem' },
          { name: 'countryId' },
          { name: 'region' },
          { name: 'city' },
        ],
        metrics: [{ name: 'screenPageViews' }],
        limit: PAGE_SIZE,
        offset,
        keepEmptyRows: false,
      })

      const rows = report.rows || []
      if (rows.length === 0) break

      for (const row of rows) {
        const dims = row.dimensionValues || []
        const metrics = row.metricValues || []

        const dateHour = dims[0]?.value || ''
        const host = normalizeGA4Value(dims[1]?.value)
        const pagePath = normalizeGA4Value(dims[2]?.value)
        const deviceRaw = dims[3]?.value || null
        const browserRaw = dims[4]?.value || null
        const osRaw = dims[5]?.value || null
        const countryId = normalizeGA4Value(dims[6]?.value)
        const regionRaw = normalizeGA4Value(dims[7]?.value)
        const cityRaw = normalizeGA4Value(dims[8]?.value)

        const count = parseInt(metrics[0]?.value || '0', 10)
        if (count <= 0) continue

        const created = this.parseGA4DateHour(dateHour)
        if (!created) continue

        const device = mapDevice(deviceRaw)
        const browser = mapBrowser(browserRaw, device)
        const os = mapOS(osRaw)
        const cc = countryId && /^[A-Z]{2}$/.test(countryId) ? countryId : null

        // Enrich with source/UTM data from lookup
        const sourceKey = `${dateHour}|${host || ''}|${pagePath || ''}`
        const sourceInfo = sourceLookup.get(sourceKey)

        const sessionSeed = [
          dateHour,
          host,
          pagePath,
          deviceRaw,
          browserRaw,
          osRaw,
          countryId,
          regionRaw,
          cityRaw,
        ].join('|')
        const psid = sessionIdToPsid(sessionSeed)

        const data: Record<string, unknown> = {
          psid,
          profileId: null,
          pid,
          host: truncate(host, 253),
          pg: truncate(pagePath, 2048),
          dv: device,
          br: truncate(browser, 30),
          brv: null,
          os: truncate(os, 25),
          osv: null,
          lc: null,
          ref: truncate(sourceInfo?.ref ?? null, 2048),
          so: truncate(sourceInfo?.so ?? null, 256),
          me: truncate(sourceInfo?.me ?? null, 256),
          ca: truncate(sourceInfo?.ca ?? null, 256),
          te: truncate(sourceInfo?.te ?? null, 256),
          co: truncate(sourceInfo?.co ?? null, 256),
          cc,
          rg: null,
          rgc: truncate(regionRaw, 100),
          ct: truncate(cityRaw, 60),
          'meta.key': [],
          'meta.value': [],
          importID,
          created,
        }

        for (let i = 0; i < count; i++) {
          yield { type: 'pageview', data }
        }
      }

      if (rows.length < PAGE_SIZE) break
      offset += PAGE_SIZE
    }
  }

  private async *fetchCustomEvents(
    client: BetaAnalyticsDataClient,
    property: string,
    dateRange: { start: string; end: string },
    pid: string,
    importID: number,
  ): AsyncIterable<AnalyticsImportRow> {
    let offset = 0

    while (true) {
      const [report] = await client.runReport({
        property,
        dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
        dimensions: [
          { name: 'dateHour' },
          { name: 'eventName' },
          { name: 'pagePath' },
          { name: 'deviceCategory' },
          { name: 'browser' },
          { name: 'operatingSystem' },
          { name: 'countryId' },
          { name: 'hostName' },
        ],
        metrics: [{ name: 'eventCount' }],
        limit: PAGE_SIZE,
        offset,
        keepEmptyRows: false,
      })

      const rows = report.rows || []
      if (rows.length === 0) break

      for (const row of rows) {
        const dims = row.dimensionValues || []
        const metrics = row.metricValues || []

        const dateHour = dims[0]?.value || ''
        const eventName = dims[1]?.value || ''
        const pagePath = normalizeGA4Value(dims[2]?.value)
        const deviceRaw = dims[3]?.value || null
        const browserRaw = dims[4]?.value || null
        const osRaw = dims[5]?.value || null
        const countryId = normalizeGA4Value(dims[6]?.value)
        const host = normalizeGA4Value(dims[7]?.value)

        if (EXCLUDED_EVENTS.has(eventName)) continue

        const count = parseInt(metrics[0]?.value || '0', 10)
        if (count <= 0) continue

        const created = this.parseGA4DateHour(dateHour)
        if (!created) continue

        const device = mapDevice(deviceRaw)
        const browser = mapBrowser(browserRaw, device)
        const os = mapOS(osRaw)
        const cc = countryId && /^[A-Z]{2}$/.test(countryId) ? countryId : null

        const sessionSeed = [
          dateHour,
          eventName,
          pagePath,
          deviceRaw,
          browserRaw,
          osRaw,
          countryId,
        ].join('|')
        const psid = sessionIdToPsid(sessionSeed)

        const data: Record<string, unknown> = {
          psid,
          profileId: null,
          pid,
          host: truncate(host, 253),
          pg: truncate(pagePath, 2048),
          event_name: truncate(eventName, 256),
          dv: device,
          br: truncate(browser, 30),
          brv: null,
          os: truncate(os, 25),
          osv: null,
          lc: null,
          ref: null,
          so: null,
          me: null,
          ca: null,
          te: null,
          co: null,
          cc,
          rg: null,
          rgc: null,
          ct: null,
          'meta.key': [],
          'meta.value': [],
          importID,
          created,
        }

        for (let i = 0; i < count; i++) {
          yield { type: 'custom_event', data }
        }
      }

      if (rows.length < PAGE_SIZE) break
      offset += PAGE_SIZE
    }
  }

  // GA4 dateHour format: "2024010115" → "2024-01-01 15:00:00"
  private parseGA4DateHour(raw: string): string | null {
    if (!raw || raw.length < 10) return null
    const year = raw.slice(0, 4)
    const month = raw.slice(4, 6)
    const day = raw.slice(6, 8)
    const hour = raw.slice(8, 10)
    return `${year}-${month}-${day} ${hour}:00:00`
  }
}