Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

7.7 KB
Raw Blame
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { Processor, WorkerHost } from '@nestjs/bullmq'
import { Logger } from '@nestjs/common'
import { Job } from 'bullmq'

import { DataImportService } from './data-import.service'
import { DataImportStatus } from './entity/data-import.entity'
import { getMapper } from './mappers'
import { ImportError } from './mappers/mapper.interface'
import { clickhouse } from '../common/integrations/clickhouse'
import { trackCustom } from '../common/analytics'

const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'
const BATCH_SIZE = 5000
const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports')

export const DATA_IMPORT_QUEUE = 'data-import'

export interface DataImportJobData {
  id: number
  importId: number
  projectId: string
  provider: string
  fileName: string
  ip: string
  userAgent: string
  // GA4-specific (API-based import)
  ga4PropertyId?: string
  encryptedRefreshToken?: string
  ga4ClientId?: string
  ga4ClientSecret?: string
}

@Processor(DATA_IMPORT_QUEUE, { concurrency: 1 })
export class DataImportProcessor extends WorkerHost {
  private readonly logger = new Logger(DataImportProcessor.name)

  constructor(private readonly dataImportService: DataImportService) {
    super()
  }

  async process(job: Job<DataImportJobData>): Promise<void> {
    const {
      id,
      importId,
      projectId,
      provider,
      fileName,
      ip,
      userAgent,
      ga4PropertyId,
      encryptedRefreshToken,
      ga4ClientId,
      ga4ClientSecret,
    } = job.data

    const isApiBased = provider === 'google-analytics'
    const filePath = isApiBased ? null : this.getImportFilePath(fileName)

    try {
      this.logger.log(
        `Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,
      )

      const dataImport = await this.dataImportService.findById(id)

      if (!dataImport) {
        this.logger.warn(`Import record ${id} not found, skipping`)
        return
      }

      if (dataImport.status === DataImportStatus.COMPLETED) {
        this.logger.log(`Import ${id} already completed, skipping reprocessing`)
        return
      }

      if (dataImport.status === DataImportStatus.PROCESSING) {
        this.logger.warn(
          `Import ${id} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`,
        )
        await this.dataImportService.cleanupImportedRows(projectId, importId)
      }

      await this.dataImportService.markProcessing(id)

      if (!isApiBased && !filePath) {
        await this.dataImportService.markFailed(
          id,
          'Invalid import file reference. Please upload the file again.',
        )
        return
      }

      const mapper = getMapper(provider)
      if (!mapper) {
        await this.dataImportService.markFailed(
          id,
          `Unsupported provider: ${provider}`,
        )
        return
      }

      let importedRows = 0
      let totalRows = 0
      let minDate: string | null = null
      let maxDate: string | null = null

      const eventsBatch: Record<string, unknown>[] = []

      const context = isApiBased
        ? {
            ga4PropertyId,
            encryptedRefreshToken,
            ga4ClientId,
            ga4ClientSecret,
          }
        : undefined

      if (isApiBased && (!ga4ClientId || !ga4ClientSecret)) {
        await this.dataImportService.markFailed(
          id,
          'GA4 import requires valid OAuth client credentials. Please reconnect your Google account.',
        )
        return
      }

      try {
        for await (const row of mapper.createRowStream(
          filePath,
          projectId,
          importId,
          context,
        )) {
          totalRows++
          if (totalRows > 100_000_000) {
            throw new ImportError(
              'Import exceeds 100 million events. Please import a shorter date range.',
            )
          }

          const created = row.data.created as string
          if (created) {
            if (!minDate || created < minDate) minDate = created
            if (!maxDate || created > maxDate) maxDate = created
          }

          eventsBatch.push({ type: row.type, ...row.data })

          if (eventsBatch.length >= BATCH_SIZE) {
            await this.flushBatch(eventsBatch)
            importedRows += eventsBatch.length
            eventsBatch.length = 0
          }

          if (totalRows % 10000 === 0) {
            const progress = {
              importedRows: importedRows + eventsBatch.length,
              totalRows,
            }

            try {
              await job.updateProgress(progress)
            } catch (progressError) {
              const errorMessage =
                progressError instanceof Error
                  ? progressError.message
                  : String(progressError)
              this.logger.error(
                `Failed to update progress for import ${id} (importId=${importId}, importedRows=${progress.importedRows}, totalRows=${progress.totalRows}): ${errorMessage}`,
              )
            }
          }
        }

        if (totalRows === 0) {
          throw new ImportError(
            'The export contains no importable events. Please check the file format and date range.',
          )
        }

        if (eventsBatch.length > 0) {
          await this.flushBatch(eventsBatch)
          importedRows += eventsBatch.length
        }

        await this.dataImportService.markCompleted(id, {
          importedRows,
          invalidRows: 0,
          totalRows: importedRows,
          dateFrom: minDate ? new Date(minDate) : null,
          dateTo: maxDate ? new Date(maxDate) : null,
        })

        this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)

        await trackCustom(ip, userAgent, {
          ev: 'DATA_IMPORT_FINISHED',
          meta: { provider, status: 'success', rows: importedRows },
        })
      } catch (error) {
        this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)

        try {
          await this.dataImportService.cleanupImportedRows(projectId, importId)
        } catch (cleanupError) {
          this.logger.error(
            `Failed to rollback partial import ${id}: ${cleanupError.message}`,
          )
        }

        const userMessage =
          error instanceof ImportError
            ? error.message
            : 'An unexpected error occurred while processing the import. Please try again or contact support.'

        await this.dataImportService.markFailed(id, userMessage)

        await trackCustom(ip, userAgent, {
          ev: 'DATA_IMPORT_FINISHED',
          meta: { provider, status: 'error', rows: importedRows },
        })
      }
    } finally {
      if (!isApiBased) {
        this.cleanupFile(fileName)
      }
    }
  }

  private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {
    await clickhouse.insert({
      table: `${CLICKHOUSE_DB}.events`,
      values: batch,
      format: 'JSONEachRow',
    })
  }

  private cleanupFile(fileName: string): void {
    try {
      const resolvedPath = this.getImportFilePath(fileName)
      if (!resolvedPath) {
        this.logger.warn(
          `Skipped cleanup for invalid import file name: ${fileName}`,
        )
        return
      }

      if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) {
        fs.unlinkSync(resolvedPath)
      }
    } catch {
      this.logger.warn(`Failed to clean up temp file: ${fileName}`)
    }
  }

  private getImportFilePath(fileName: string): string | null {
    if (
      !fileName ||
      fileName !== path.basename(fileName) ||
      /[\\/]/.test(fileName)
    ) {
      return null
    }

    return path.join(IMPORT_TMP_DIR, fileName)
  }
}