📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import { Processor, WorkerHost } from '@nestjs/bullmq'
import { Logger } from '@nestjs/common'
import { Job } from 'bullmq'
import { DataImportService } from './data-import.service'
import { DataImportStatus } from './entity/data-import.entity'
import { getMapper } from './mappers'
import { ImportError } from './mappers/mapper.interface'
import { clickhouse } from '../common/integrations/clickhouse'
import { trackCustom } from '../common/analytics'
const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'
const BATCH_SIZE = 5000
const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports')
export const DATA_IMPORT_QUEUE = 'data-import'
export interface DataImportJobData {
id: number
importId: number
projectId: string
provider: string
fileName: string
ip: string
userAgent: string
// GA4-specific (API-based import)
ga4PropertyId?: string
encryptedRefreshToken?: string
ga4ClientId?: string
ga4ClientSecret?: string
}
@Processor(DATA_IMPORT_QUEUE, { concurrency: 1 })
export class DataImportProcessor extends WorkerHost {
private readonly logger = new Logger(DataImportProcessor.name)
constructor(private readonly dataImportService: DataImportService) {
super()
}
async process(job: Job<DataImportJobData>): Promise<void> {
const {
id,
importId,
projectId,
provider,
fileName,
ip,
userAgent,
ga4PropertyId,
encryptedRefreshToken,
ga4ClientId,
ga4ClientSecret,
} = job.data
const isApiBased = provider === 'google-analytics'
const filePath = isApiBased ? null : this.getImportFilePath(fileName)
try {
this.logger.log(
`Processing import ${id} (importId=${importId}) for project ${projectId} (provider: ${provider})`,
)
const dataImport = await this.dataImportService.findById(id)
if (!dataImport) {
this.logger.warn(`Import record ${id} not found, skipping`)
return
}
if (dataImport.status === DataImportStatus.COMPLETED) {
this.logger.log(`Import ${id} already completed, skipping reprocessing`)
return
}
if (dataImport.status === DataImportStatus.PROCESSING) {
this.logger.warn(
`Import ${id} has status PROCESSING (stalled job replay), cleaning up partial data before reprocessing`,
)
await this.dataImportService.cleanupImportedRows(projectId, importId)
}
await this.dataImportService.markProcessing(id)
if (!isApiBased && !filePath) {
await this.dataImportService.markFailed(
id,
'Invalid import file reference. Please upload the file again.',
)
return
}
const mapper = getMapper(provider)
if (!mapper) {
await this.dataImportService.markFailed(
id,
`Unsupported provider: ${provider}`,
)
return
}
let importedRows = 0
let totalRows = 0
let minDate: string | null = null
let maxDate: string | null = null
const eventsBatch: Record<string, unknown>[] = []
const context = isApiBased
? {
ga4PropertyId,
encryptedRefreshToken,
ga4ClientId,
ga4ClientSecret,
}
: undefined
if (isApiBased && (!ga4ClientId || !ga4ClientSecret)) {
await this.dataImportService.markFailed(
id,
'GA4 import requires valid OAuth client credentials. Please reconnect your Google account.',
)
return
}
try {
for await (const row of mapper.createRowStream(
filePath,
projectId,
importId,
context,
)) {
totalRows++
if (totalRows > 100_000_000) {
throw new ImportError(
'Import exceeds 100 million events. Please import a shorter date range.',
)
}
const created = row.data.created as string
if (created) {
if (!minDate || created < minDate) minDate = created
if (!maxDate || created > maxDate) maxDate = created
}
eventsBatch.push({ type: row.type, ...row.data })
if (eventsBatch.length >= BATCH_SIZE) {
await this.flushBatch(eventsBatch)
importedRows += eventsBatch.length
eventsBatch.length = 0
}
if (totalRows % 10000 === 0) {
const progress = {
importedRows: importedRows + eventsBatch.length,
totalRows,
}
try {
await job.updateProgress(progress)
} catch (progressError) {
const errorMessage =
progressError instanceof Error
? progressError.message
: String(progressError)
this.logger.error(
`Failed to update progress for import ${id} (importId=${importId}, importedRows=${progress.importedRows}, totalRows=${progress.totalRows}): ${errorMessage}`,
)
}
}
}
if (totalRows === 0) {
throw new ImportError(
'The export contains no importable events. Please check the file format and date range.',
)
}
if (eventsBatch.length > 0) {
await this.flushBatch(eventsBatch)
importedRows += eventsBatch.length
}
await this.dataImportService.markCompleted(id, {
importedRows,
invalidRows: 0,
totalRows: importedRows,
dateFrom: minDate ? new Date(minDate) : null,
dateTo: maxDate ? new Date(maxDate) : null,
})
this.logger.log(`Import ${id} completed: ${importedRows} rows imported`)
await trackCustom(ip, userAgent, {
ev: 'DATA_IMPORT_FINISHED',
meta: { provider, status: 'success', rows: importedRows },
})
} catch (error) {
this.logger.error(`Import ${id} failed: ${error.message}`, error.stack)
try {
await this.dataImportService.cleanupImportedRows(projectId, importId)
} catch (cleanupError) {
this.logger.error(
`Failed to rollback partial import ${id}: ${cleanupError.message}`,
)
}
const userMessage =
error instanceof ImportError
? error.message
: 'An unexpected error occurred while processing the import. Please try again or contact support.'
await this.dataImportService.markFailed(id, userMessage)
await trackCustom(ip, userAgent, {
ev: 'DATA_IMPORT_FINISHED',
meta: { provider, status: 'error', rows: importedRows },
})
}
} finally {
if (!isApiBased) {
this.cleanupFile(fileName)
}
}
}
private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {
await clickhouse.insert({
table: `${CLICKHOUSE_DB}.events`,
values: batch,
format: 'JSONEachRow',
})
}
private cleanupFile(fileName: string): void {
try {
const resolvedPath = this.getImportFilePath(fileName)
if (!resolvedPath) {
this.logger.warn(
`Skipped cleanup for invalid import file name: ${fileName}`,
)
return
}
if (fs.existsSync(resolvedPath) && fs.lstatSync(resolvedPath).isFile()) {
fs.unlinkSync(resolvedPath)
}
} catch {
this.logger.warn(`Failed to clean up temp file: ${fileName}`)
}
}
private getImportFilePath(fileName: string): string | null {
if (
!fileName ||
fileName !== path.basename(fileName) ||
/[\\/]/.test(fileName)
) {
return null
}
return path.join(IMPORT_TMP_DIR, fileName)
}
}