Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Swetrix CE - Google Analytics importer

Blue Mouse0bd430ff1569
diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.ts
index 359987b7c..53d5f9f0a 100644
--- a/backend/apps/community/src/common/utils.ts
+++ b/backend/apps/community/src/common/utils.ts
@@ -85,7 +85,7 @@ export const hash = (content: string): string => {
* Derives a key from the SECRET_KEY_BASE environment variable using the HKDF algorithm.
*/
export const deriveKey = (
- purpose: 'refresh-token' | 'access-token' | 'gsc-token',
+ purpose: 'refresh-token' | 'access-token' | 'gsc-token' | 'ga4-token',
length = 32,
) => {
const base = Buffer.from(process.env.SECRET_KEY_BASE || '', 'utf8')
diff --git a/backend/apps/community/src/data-import/data-import.controller.ts b/backend/apps/community/src/data-import/data-import.controller.ts
index ed7b259d1..3178a840a 100644
--- a/backend/apps/community/src/data-import/data-import.controller.ts
+++ b/backend/apps/community/src/data-import/data-import.controller.ts
@@ -16,6 +16,8 @@ import {
ServiceUnavailableException,
Query,
Logger,
+ Headers,
+ Ip,
} from '@nestjs/common'
import { FileInterceptor } from '@nestjs/platform-express'
import { diskStorage } from 'multer'
@@ -27,9 +29,11 @@ import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { Auth } from '../auth/decorators'
import { ProjectService } from '../project/project.service'
import { DataImportService } from './data-import.service'
-import { UploadImportDto } from './dto'
+import { Ga4ImportService, Ga4Property } from './ga4-import.service'
+import { UploadImportDto, StartGa4ImportDto } from './dto'
import { getMapper, SUPPORTED_PROVIDERS } from './mappers'
import { DataImportJobData, DATA_IMPORT_QUEUE } from './data-import.processor'
+import { getIPFromHeaders } from '../common/utils'
const MAX_FILE_SIZE = 100 * 1024 * 1024 // 100 MB
const IMPORT_TMP_DIR = path.resolve(os.tmpdir(), 'swetrix-imports')
@@ -42,10 +46,104 @@ export class DataImportController {
constructor(
private readonly dataImportService: DataImportService,
private readonly projectService: ProjectService,
+ private readonly ga4ImportService: Ga4ImportService,
@InjectQueue(DATA_IMPORT_QUEUE)
private readonly importQueue: Queue<DataImportJobData>,
) {}
+ @Get('ga4/configured')
+ @Auth()
+ async ga4Configured() {
+ return { configured: this.ga4ImportService.isConfigured() }
+ }
+
+ @Post(':projectId/ga4/connect')
+ @Auth()
+ async ga4Connect(
+ @Param('projectId') projectId: string,
+ @CurrentUserId() uid: string,
+ ) {
+ return this.ga4ImportService.generateConnectURL(uid, projectId)
+ }
+
+ @Post('ga4/callback')
+ @Auth()
+ async ga4Callback(
+ @CurrentUserId() uid: string,
+ @Body() body: { code: string; state: string },
+ ) {
+ if (!body.code || !body.state) {
+ throw new BadRequestException('Missing code or state parameter')
+ }
+ return this.ga4ImportService.handleOAuthCallback(uid, body.code, body.state)
+ }
+
+ @Get(':projectId/ga4/properties')
+ @Auth()
+ async ga4Properties(
+ @Param('projectId') projectId: string,
+ @CurrentUserId() uid: string,
+ ): Promise<Ga4Property[]> {
+ return this.ga4ImportService.listProperties(uid, projectId)
+ }
+
+ @Post(':projectId/ga4/start')
+ @Auth()
+ async ga4Start(
+ @Param('projectId') projectId: string,
+ @CurrentUserId() uid: string,
+ @Body() dto: StartGa4ImportDto,
+ @Headers() headers: Record<string, string>,
+ @Ip() requestIp: string,
+ ) {
+ const project = await this.projectService.getFullProject(projectId)
+ this.projectService.allowedToManage(project, uid)
+
+ const dataImport = await this.dataImportService.create(
+ projectId,
+ 'google-analytics',
+ )
+
+ const { encryptedRefreshToken, clientId, clientSecret } =
+ await this.ga4ImportService.consumeTokenForImport(uid, projectId)
+
+ try {
+ await this.importQueue.add(
+ 'process-import',
+ {
+ importId: dataImport.id,
+ projectId,
+ provider: 'google-analytics',
+ fileName: '',
+ ip: getIPFromHeaders(headers) || requestIp || '',
+ userAgent: headers['user-agent'] || '',
+ ga4PropertyId: dto.propertyId,
+ encryptedRefreshToken,
+ ga4ClientId: clientId,
+ ga4ClientSecret: clientSecret,
+ },
+ {
+ attempts: 1,
+ removeOnComplete: true,
+ removeOnFail: false,
+ },
+ )
+ } catch (error) {
+ this.logger.error(`Failed to enqueue GA4 import job: ${error.message}`)
+ await this.dataImportService.deleteImportRecord(projectId, dataImport.id)
+ await this.ga4ImportService.restoreToken(
+ uid,
+ projectId,
+ encryptedRefreshToken,
+ )
+ throw new ServiceUnavailableException(
+ 'Failed to start import. Please try again later.',
+ )
+ }
+
+ return dataImport
+ }
+
@Post(':projectId/upload')
@Auth()
@ApiConsumes('multipart/form-data')
@@ -94,6 +192,13 @@ export class DataImportController {
)
}
+ if (!mapper.expectedFileExtension) {
+ this.cleanupFile(file.filename)
+ throw new BadRequestException(
+ `${dto.provider} does not support file uploads. Use the dedicated import endpoint instead.`,
+ )
+ }
+
const ext = path.extname(file.originalname).toLowerCase()
if (ext !== mapper.expectedFileExtension) {
this.cleanupFile(file.filename)
diff --git a/backend/apps/community/src/data-import/data-import.module.ts b/backend/apps/community/src/data-import/data-import.module.ts
index 90b6b48ee..ec6c24edc 100644
--- a/backend/apps/community/src/data-import/data-import.module.ts
+++ b/backend/apps/community/src/data-import/data-import.module.ts
@@ -3,6 +3,7 @@ import { BullModule } from '@nestjs/bullmq'
import { ProjectModule } from '../project/project.module'
import { DataImportService } from './data-import.service'
+import { Ga4ImportService } from './ga4-import.service'
import { DataImportController } from './data-import.controller'
import { DataImportProcessor, DATA_IMPORT_QUEUE } from './data-import.processor'
@@ -11,7 +12,7 @@ import { DataImportProcessor, DATA_IMPORT_QUEUE } from './data-import.processor'
BullModule.registerQueue({ name: DATA_IMPORT_QUEUE }),
forwardRef(() => ProjectModule),
],
- providers: [DataImportService, DataImportProcessor],
+ providers: [DataImportService, Ga4ImportService, DataImportProcessor],
controllers: [DataImportController],
exports: [DataImportService],
})
diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts
index 5f3ebfe97..9ebde1d6e 100644
--- a/backend/apps/community/src/data-import/data-import.processor.ts
+++ b/backend/apps/community/src/data-import/data-import.processor.ts
@@ -22,6 +22,13 @@ export interface DataImportJobData {
projectId: string
provider: string
fileName: string
+ ip?: string
+ userAgent?: string
+ // GA4-specific (API-based import)
+ ga4PropertyId?: string
+ encryptedRefreshToken?: string
+ ga4ClientId?: string
+ ga4ClientSecret?: string
}
@Processor(DATA_IMPORT_QUEUE, { concurrency: 1 })
@@ -33,8 +40,18 @@ export class DataImportProcessor extends WorkerHost {
}
async process(job: Job<DataImportJobData>): Promise<void> {
- const { importId, projectId, provider, fileName } = job.data
- const filePath = this.getImportFilePath(fileName)
+ const {
+ importId,
+ projectId,
+ provider,
+ fileName,
+ ga4PropertyId,
+ encryptedRefreshToken,
+ ga4ClientId,
+ ga4ClientSecret,
+ } = job.data
+ const isApiBased = provider === 'google-analytics'
+ const filePath = isApiBased ? null : this.getImportFilePath(fileName)
try {
this.logger.log(
@@ -62,7 +79,7 @@ export class DataImportProcessor extends WorkerHost {
await this.dataImportService.markProcessing(importId, projectId)
- if (!filePath) {
+ if (!isApiBased && !filePath) {
await this.dataImportService.markFailed(
importId,
projectId,
@@ -88,11 +105,30 @@ export class DataImportProcessor extends WorkerHost {
const eventsBatch: Record<string, unknown>[] = []
+ const context = isApiBased
+ ? {
+ ga4PropertyId,
+ encryptedRefreshToken,
+ ga4ClientId,
+ ga4ClientSecret,
+ }
+ : undefined
+
+ if (isApiBased && (!ga4ClientId || !ga4ClientSecret)) {
+ await this.dataImportService.markFailed(
+ importId,
+ projectId,
+ 'GA4 import requires valid OAuth client credentials. Please reconnect your Google account.',
+ )
+ return
+ }
+
try {
for await (const row of mapper.createRowStream(
filePath,
projectId,
importId,
+ context,
)) {
totalRows++
@@ -172,7 +208,9 @@ export class DataImportProcessor extends WorkerHost {
)
}
} finally {
- this.cleanupFile(fileName)
+ if (!isApiBased) {
+ this.cleanupFile(fileName)
+ }
}
}
diff --git a/backend/apps/community/src/data-import/dto/index.ts b/backend/apps/community/src/data-import/dto/index.ts
index 0221d0e71..4eaf9ef6b 100644
--- a/backend/apps/community/src/data-import/dto/index.ts
+++ b/backend/apps/community/src/data-import/dto/index.ts
@@ -1 +1,2 @@
export { UploadImportDto } from './upload-import.dto'
+export { StartGa4ImportDto } from './start-ga4-import.dto'
diff --git a/backend/apps/community/src/data-import/dto/start-ga4-import.dto.ts b/backend/apps/community/src/data-import/dto/start-ga4-import.dto.ts
new file mode 100644
index 000000000..a5052a233
--- /dev/null
+++ b/backend/apps/community/src/data-import/dto/start-ga4-import.dto.ts
@@ -0,0 +1,12 @@
+import { IsNumberString } from 'class-validator'
+import { ApiProperty } from '@nestjs/swagger'
+
+export class StartGa4ImportDto {
+ @ApiProperty({
+ description:
+ 'GA4 property ID (numeric ID, not the "properties/" prefixed form)',
+ example: '123456789',
+ })
+ @IsNumberString()
+ propertyId: string
+}
diff --git a/backend/apps/community/src/data-import/ga4-import.service.ts b/backend/apps/community/src/data-import/ga4-import.service.ts
new file mode 100644
index 000000000..eb90522f9
--- /dev/null
+++ b/backend/apps/community/src/data-import/ga4-import.service.ts
@@ -0,0 +1,288 @@
+import {
+ Injectable,
+ BadRequestException,
+ InternalServerErrorException,
+} from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
+import { OAuth2Client } from 'google-auth-library'
+import * as crypto from 'crypto'
+
+import { redis } from '../common/constants'
+import { ProjectService } from '../project/project.service'
+import { deriveKey } from '../common/utils'
+
+const REDIS_STATE_PREFIX = 'ga4-import:state:'
+const REDIS_TOKEN_PREFIX = 'ga4-import:token:'
+const REDIS_TOKEN_TTL = 600 // 10 minutes
+
+const ENCRYPTION_KEY = Buffer.from(deriveKey('ga4-token', 32), 'hex')
+
+function encryptToken(plaintext: string): string {
+ const iv = crypto.randomBytes(12)
+ const cipher = crypto.createCipheriv('aes-256-gcm', ENCRYPTION_KEY, iv)
+ const encrypted = Buffer.concat([
+ cipher.update(plaintext, 'utf8'),
+ cipher.final(),
+ ])
+ const authTag = cipher.getAuthTag()
+ return `${iv.toString('hex')}:${encrypted.toString('hex')}:${authTag.toString('hex')}`
+}
+
+function decryptToken(ciphertext: string): string {
+ const parts = ciphertext.split(':')
+ if (parts.length !== 3) {
+ throw new Error('Invalid encrypted token format')
+ }
+ const [ivHex, encHex, tagHex] = parts
+ const decipher = crypto.createDecipheriv(
+ 'aes-256-gcm',
+ ENCRYPTION_KEY,
+ Buffer.from(ivHex, 'hex'),
+ )
+ decipher.setAuthTag(Buffer.from(tagHex, 'hex'))
+ const decrypted = Buffer.concat([
+ decipher.update(Buffer.from(encHex, 'hex')),
+ decipher.final(),
+ ])
+ return decrypted.toString('utf8')
+}
+
+const GA4_SCOPES = ['https://www.googleapis.com/auth/analytics.readonly']
+
+const getBaseUrl = (): string | null => {
+ const raw = process.env.BASE_URL?.trim()
+ if (!raw) return null
+ return raw.replace(/\/+$/, '')
+}
+
+export interface Ga4Property {
+ propertyId: string
+ displayName: string
+}
+
+@Injectable()
+export class Ga4ImportService {
+ constructor(
+ private readonly configService: ConfigService,
+ private readonly projectService: ProjectService,
+ ) {}
+
+ private getCredentials() {
+ const clientId = this.configService.get<string>('GOOGLE_GA4_CLIENT_ID')
+ const clientSecret = this.configService.get<string>(
+ 'GOOGLE_GA4_CLIENT_SECRET',
+ )
+
+ if (!clientId || !clientSecret) {
+ throw new InternalServerErrorException(
+ 'Google Analytics import is not configured on this instance. Set GOOGLE_GA4_CLIENT_ID and GOOGLE_GA4_CLIENT_SECRET on the API container.',
+ )
+ }
+
+ return { clientId, clientSecret }
+ }
+
+ private getRedirectUrl(): string {
+ const base = getBaseUrl()
+ if (!base) {
+ throw new InternalServerErrorException(
+ 'BASE_URL is not configured. Set BASE_URL on the API container to enable Google Analytics import.',
+ )
+ }
+ return `${base}/ga4-import-connected`
+ }
+
+ private getOAuthClient() {
+ const { clientId, clientSecret } = this.getCredentials()
+ return new OAuth2Client(clientId, clientSecret, this.getRedirectUrl())
+ }
+
+ async generateConnectURL(uid: string, pid: string): Promise<{ url: string }> {
+ const project = await this.projectService.getFullProject(pid)
+ this.projectService.allowedToManage(project, uid)
+
+ const oauth2Client = this.getOAuthClient()
+
+ const state = crypto.randomBytes(32).toString('hex')
+ await redis.set(
+ REDIS_STATE_PREFIX + state,
+ JSON.stringify({ uid, pid }),
+ 'EX',
+ REDIS_TOKEN_TTL,
+ )
+
+ const url = oauth2Client.generateAuthUrl({
+ access_type: 'offline',
+ scope: GA4_SCOPES,
+ prompt: 'consent',
+ state,
+ })
+
+ return { url }
+ }
+
+ async handleOAuthCallback(
+ uid: string,
+ code: string,
+ state: string,
+ ): Promise<{ pid: string }> {
+ const cached = await redis.get(REDIS_STATE_PREFIX + state)
+ if (!cached) {
+ throw new BadRequestException('Invalid or expired OAuth state')
+ }
+
+ const { pid, uid: stateUid } = JSON.parse(cached) as {
+ pid: string
+ uid: string
+ }
+
+ if (stateUid !== uid) {
+ throw new BadRequestException('OAuth state mismatch')
+ }
+
+ const project = await this.projectService.getFullProject(pid)
+ this.projectService.allowedToManage(project, uid)
+
+ const oauth2Client = this.getOAuthClient()
+ const { tokens } = await oauth2Client.getToken(code)
+
+ if (!tokens.refresh_token) {
+ throw new BadRequestException(
+ 'Google did not return a refresh token. Please try connecting again.',
+ )
+ }
+
+ const encrypted = encryptToken(tokens.refresh_token)
+
+ await redis.set(
+ REDIS_TOKEN_PREFIX + `${uid}:${pid}`,
+ encrypted,
+ 'EX',
+ REDIS_TOKEN_TTL,
+ )
+
+ await redis.del(REDIS_STATE_PREFIX + state)
+
+ return { pid }
+ }
+
+ async listProperties(uid: string, pid: string): Promise<Ga4Property[]> {
+ const project = await this.projectService.getFullProject(pid)
+ this.projectService.allowedToManage(project, uid)
+
+ const encrypted = await redis.get(REDIS_TOKEN_PREFIX + `${uid}:${pid}`)
+ if (!encrypted) {
+ throw new BadRequestException(
+ 'Google Analytics connection expired. Please connect again.',
+ )
+ }
+
+ const refreshToken = decryptToken(encrypted)
+
+ const oauth2Client = this.getOAuthClient()
+ oauth2Client.setCredentials({ refresh_token: refreshToken })
+
+ const { token } = await oauth2Client.getAccessToken()
+ if (!token) {
+ throw new InternalServerErrorException(
+ 'Failed to obtain access token from Google',
+ )
+ }
+
+ // Refresh the TTL since the user is actively using the connection
+ await redis.expire(REDIS_TOKEN_PREFIX + `${uid}:${pid}`, REDIS_TOKEN_TTL)
+
+ try {
+ const url = new URL(
+ 'https://analyticsadmin.googleapis.com/v1beta/accountSummaries',
+ )
+ url.searchParams.set('pageSize', '200')
+
+ const response = await fetch(url, {
+ headers: { Authorization: `Bearer ${token}` },
+ })
+
+ if (response.status === 403) {
+ const body = await response.json()
+ throw new BadRequestException(
+ body?.error?.message ||
+ 'Access denied by Google. Please ensure the "Google Analytics Admin API" is enabled in your Google Cloud project and that your Google account has access to at least one GA4 property.',
+ )
+ }
+
+ if (!response.ok) {
+ throw new Error(`Google API responded with status ${response.status}`)
+ }
+
+ const data = await response.json()
+ const properties: Ga4Property[] = []
+
+ for (const account of data.accountSummaries || []) {
+ for (const prop of account.propertySummaries || []) {
+ properties.push({
+ propertyId: prop.property?.replace('properties/', '') || '',
+ displayName: prop.displayName || prop.property || 'Unknown',
+ })
+ }
+ }
+
+ return properties
+ } catch (error) {
+ if (error instanceof BadRequestException) throw error
+
+ throw new InternalServerErrorException(
+ 'Failed to fetch Google Analytics properties. Please try again.',
+ )
+ }
+ }
+
+ consumeTokenForImport(
+ uid: string,
+ pid: string,
+ ): Promise<{
+ encryptedRefreshToken: string
+ clientId: string
+ clientSecret: string
+ }> {
+ return this.getAndDeleteToken(uid, pid)
+ }
+
+ async restoreToken(
+ uid: string,
+ pid: string,
+ encryptedRefreshToken: string,
+ ): Promise<void> {
+ const key = REDIS_TOKEN_PREFIX + `${uid}:${pid}`
+ await redis.set(key, encryptedRefreshToken, 'EX', REDIS_TOKEN_TTL)
+ }
+
+ private async getAndDeleteToken(
+ uid: string,
+ pid: string,
+ ): Promise<{
+ encryptedRefreshToken: string
+ clientId: string
+ clientSecret: string
+ }> {
+ const key = REDIS_TOKEN_PREFIX + `${uid}:${pid}`
+ const encrypted = await redis.getdel(key)
+
+ if (!encrypted) {
+ throw new BadRequestException(
+ 'Google Analytics connection expired. Please connect again.',
+ )
+ }
+
+ const { clientId, clientSecret } = this.getCredentials()
+
+ return { encryptedRefreshToken: encrypted, clientId, clientSecret }
+ }
+
+ isConfigured(): boolean {
+ const clientId = this.configService.get<string>('GOOGLE_GA4_CLIENT_ID')
+ const clientSecret = this.configService.get<string>(
+ 'GOOGLE_GA4_CLIENT_SECRET',
+ )
+ return !!(clientId && clientSecret && getBaseUrl())
+ }
+}
diff --git a/backend/apps/community/src/data-import/mappers/ga4.mapper.ts b/backend/apps/community/src/data-import/mappers/ga4.mapper.ts
new file mode 100644
index 000000000..2ee657aa9
--- /dev/null
+++ b/backend/apps/community/src/data-import/mappers/ga4.mapper.ts
@@ -0,0 +1,486 @@
+import { BetaAnalyticsDataClient } from '@google-analytics/data'
+import { OAuth2Client } from 'google-auth-library'
+import * as crypto from 'crypto'
+import dayjs from 'dayjs'
+
+import {
+ ImportMapper,
+ ImportError,
+ AnalyticsImportRow,
+} from './mapper.interface'
+import {
+ truncate,
+ sessionIdToPsid,
+ MOBILE_BROWSER_VARIANTS,
+} from './mapper.utils'
+import { deriveKey } from '../../common/utils'
+
+const ENCRYPTION_KEY = Buffer.from(deriveKey('ga4-token', 32), 'hex')
+
+const PAGE_SIZE = 100_000
+
+const EXCLUDED_EVENTS = new Set([
+ 'page_view',
+ 'session_start',
+ 'first_visit',
+ 'scroll',
+ 'click',
+ 'file_download',
+ 'form_start',
+ 'form_submit',
+ 'view_search_results',
+ 'video_start',
+ 'video_progress',
+ 'video_complete',
+])
+
+const DEVICE_MAP: Record<string, string> = {
+ desktop: 'desktop',
+ mobile: 'mobile',
+ tablet: 'tablet',
+}
+
+function mapDevice(raw: string | null): string {
+ if (!raw) return 'desktop'
+ return DEVICE_MAP[raw.toLowerCase()] ?? 'desktop'
+}
+
+function mapBrowser(raw: string | null, device: string): string | null {
+ if (!raw || raw === '(not set)') return null
+ const isMobile = device === 'mobile' || device === 'tablet'
+ if (isMobile && MOBILE_BROWSER_VARIANTS[raw]) {
+ return MOBILE_BROWSER_VARIANTS[raw]
+ }
+ return raw
+}
+
+function mapOS(raw: string | null): string | null {
+ if (!raw || raw === '(not set)') return null
+ if (raw === 'Macintosh') return 'macOS'
+ return raw
+}
+
+function normalizeGA4Value(val: string | null | undefined): string | null {
+ if (!val || val === '(not set)' || val === '(none)' || val === '(direct)') {
+ return null
+ }
+ return val
+}
+
+function buildReferrer(source: string | null): string | null {
+ if (!source) return null
+ if (source.includes('://')) return source
+ if (source.includes('.')) return `https://${source}`
+ return null
+}
+
+function decryptToken(ciphertext: string): string {
+ const parts = ciphertext.split(':')
+ if (parts.length !== 3) {
+ throw new Error('Invalid encrypted token format')
+ }
+ const [ivHex, encHex, tagHex] = parts
+ const decipher = crypto.createDecipheriv(
+ 'aes-256-gcm',
+ ENCRYPTION_KEY,
+ Buffer.from(ivHex, 'hex'),
+ )
+ decipher.setAuthTag(Buffer.from(tagHex, 'hex'))
+ const decrypted = Buffer.concat([
+ decipher.update(Buffer.from(encHex, 'hex')),
+ decipher.final(),
+ ])
+ return decrypted.toString('utf8')
+}
+
+function getMonthChunks(
+ startDate: string,
+ endDate: string,
+): { start: string; end: string }[] {
+ const chunks: { start: string; end: string }[] = []
+ let current = dayjs(startDate)
+ const end = dayjs(endDate)
+
+ while (current.isBefore(end) || current.isSame(end, 'day')) {
+ const monthEnd = current.endOf('month')
+ const chunkEnd = monthEnd.isAfter(end) ? end : monthEnd
+ chunks.push({
+ start: current.format('YYYY-MM-DD'),
+ end: chunkEnd.format('YYYY-MM-DD'),
+ })
+ current = chunkEnd.add(1, 'day')
+ }
+
+ return chunks
+}
+
+interface SourceInfo {
+ so: string | null
+ me: string | null
+ ca: string | null
+ te: string | null
+ co: string | null
+ ref: string | null
+}
+
+export class Ga4Mapper implements ImportMapper {
+ readonly provider = 'google-analytics'
+ readonly expectedFileExtension = null
+
+ async *createRowStream(
+ _filePath: string | null,
+ pid: string,
+ importID: number,
+ context?: Record<string, unknown>,
+ ): AsyncIterable<AnalyticsImportRow> {
+ if (
+ !context?.encryptedRefreshToken ||
+ !context?.ga4PropertyId ||
+ !context?.ga4ClientId ||
+ !context?.ga4ClientSecret
+ ) {
+ throw new ImportError(
+ 'GA4 import requires OAuth credentials and a property ID.',
+ )
+ }
+
+ const refreshToken = decryptToken(context.encryptedRefreshToken as string)
+ const propertyId = context.ga4PropertyId as string
+ const clientId = context.ga4ClientId as string
+ const clientSecret = context.ga4ClientSecret as string
+
+ const oauth2Client = new OAuth2Client(clientId, clientSecret)
+ oauth2Client.setCredentials({ refresh_token: refreshToken })
+
+ const client = new BetaAnalyticsDataClient({
+ authClient: oauth2Client as any,
+ })
+ const property = propertyId.startsWith('properties/')
+ ? propertyId
+ : `properties/${propertyId}`
+
+ const startDate = (context.startDate as string) || '2020-01-01'
+ const endDate = (context.endDate as string) || dayjs().format('YYYY-MM-DD')
+ const chunks = getMonthChunks(startDate, endDate)
+
+ for (const chunk of chunks) {
+ const sourceLookup = await this.buildSourceLookup(client, property, chunk)
+ yield* this.fetchPageviews(
+ client,
+ property,
+ chunk,
+ pid,
+ importID,
+ sourceLookup,
+ )
+ yield* this.fetchCustomEvents(client, property, chunk, pid, importID)
+ }
+ }
+
+ /**
+ * Fetches source/UTM data into a lookup map keyed by (dateHour|hostName|pagePath).
+ * Kept in memory per monthly chunk -- the map stores only the highest-traffic
+ * source combo for each key, so memory stays bounded.
+ */
+ private async buildSourceLookup(
+ client: BetaAnalyticsDataClient,
+ property: string,
+ dateRange: { start: string; end: string },
+ ): Promise<Map<string, SourceInfo>> {
+ const lookup = new Map<string, SourceInfo & { count: number }>()
+ let offset = 0
+
+ while (true) {
+ const [report] = await client.runReport({
+ property,
+ dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
+ dimensions: [
+ { name: 'dateHour' },
+ { name: 'hostName' },
+ { name: 'pagePath' },
+ { name: 'sessionSource' },
+ { name: 'sessionMedium' },
+ { name: 'sessionCampaignName' },
+ { name: 'sessionManualTerm' },
+ { name: 'sessionManualAdContent' },
+ ],
+ metrics: [{ name: 'screenPageViews' }],
+ limit: PAGE_SIZE,
+ offset,
+ keepEmptyRows: false,
+ })
+
+ const rows = report.rows || []
+ if (rows.length === 0) break
+
+ for (const row of rows) {
+ const dims = row.dimensionValues || []
+ const metrics = row.metricValues || []
+
+ const dateHour = dims[0]?.value || ''
+ const host = normalizeGA4Value(dims[1]?.value)
+ const pagePath = normalizeGA4Value(dims[2]?.value)
+ const source = normalizeGA4Value(dims[3]?.value)
+ const medium = normalizeGA4Value(dims[4]?.value)
+ const campaign = normalizeGA4Value(dims[5]?.value)
+ const term = normalizeGA4Value(dims[6]?.value)
+ const content = normalizeGA4Value(dims[7]?.value)
+
+ const count = parseInt(metrics[0]?.value || '0', 10)
+ if (count <= 0) continue
+
+ const key = `${dateHour}|${host || ''}|${pagePath || ''}`
+ const existing = lookup.get(key)
+
+ if (!existing || count > existing.count) {
+ lookup.set(key, {
+ so: source,
+ me: medium,
+ ca: campaign,
+ te: term,
+ co: content,
+ ref: buildReferrer(source),
+ count,
+ })
+ }
+ }
+
+ if (rows.length < PAGE_SIZE) break
+ offset += PAGE_SIZE
+ }
+
+ return lookup
+ }
+
+ private async *fetchPageviews(
+ client: BetaAnalyticsDataClient,
+ property: string,
+ dateRange: { start: string; end: string },
+ pid: string,
+ importID: number,
+ sourceLookup: Map<string, SourceInfo>,
+ ): AsyncIterable<AnalyticsImportRow> {
+ let offset = 0
+
+ while (true) {
+ const [report] = await client.runReport({
+ property,
+ dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
+ dimensions: [
+ { name: 'dateHour' },
+ { name: 'hostName' },
+ { name: 'pagePath' },
+ { name: 'deviceCategory' },
+ { name: 'browser' },
+ { name: 'operatingSystem' },
+ { name: 'countryId' },
+ { name: 'region' },
+ { name: 'city' },
+ ],
+ metrics: [{ name: 'screenPageViews' }],
+ limit: PAGE_SIZE,
+ offset,
+ keepEmptyRows: false,
+ })
+
+ const rows = report.rows || []
+ if (rows.length === 0) break
+
+ for (const row of rows) {
+ const dims = row.dimensionValues || []
+ const metrics = row.metricValues || []
+
+ const dateHour = dims[0]?.value || ''
+ const host = normalizeGA4Value(dims[1]?.value)
+ const pagePath = normalizeGA4Value(dims[2]?.value)
+ const deviceRaw = dims[3]?.value || null
+ const browserRaw = dims[4]?.value || null
+ const osRaw = dims[5]?.value || null
+ const countryId = normalizeGA4Value(dims[6]?.value)
+ const regionRaw = normalizeGA4Value(dims[7]?.value)
+ const cityRaw = normalizeGA4Value(dims[8]?.value)
+
+ const count = parseInt(metrics[0]?.value || '0', 10)
+ if (count <= 0) continue
+
+ const created = this.parseGA4DateHour(dateHour)
+ if (!created) continue
+
+ const device = mapDevice(deviceRaw)
+ const browser = mapBrowser(browserRaw, device)
+ const os = mapOS(osRaw)
+ const cc = countryId && /^[A-Z]{2}$/.test(countryId) ? countryId : null
+
+ // Enrich with source/UTM data from lookup
+ const sourceKey = `${dateHour}|${host || ''}|${pagePath || ''}`
+ const sourceInfo = sourceLookup.get(sourceKey)
+
+ const sessionSeed = [
+ dateHour,
+ host,
+ pagePath,
+ deviceRaw,
+ browserRaw,
+ osRaw,
+ countryId,
+ regionRaw,
+ cityRaw,
+ ].join('|')
+ const psid = sessionIdToPsid(sessionSeed)
+
+ const data: Record<string, unknown> = {
+ psid,
+ profileId: null,
+ pid,
+ host: truncate(host, 253),
+ pg: truncate(pagePath, 2048),
+ dv: device,
+ br: truncate(browser, 30),
+ brv: null,
+ os: truncate(os, 25),
+ osv: null,
+ lc: null,
+ ref: truncate(sourceInfo?.ref ?? null, 2048),
+ so: truncate(sourceInfo?.so ?? null, 256),
+ me: truncate(sourceInfo?.me ?? null, 256),
+ ca: truncate(sourceInfo?.ca ?? null, 256),
+ te: truncate(sourceInfo?.te ?? null, 256),
+ co: truncate(sourceInfo?.co ?? null, 256),
+ cc,
+ rg: null,
+ rgc: truncate(regionRaw, 100),
+ ct: truncate(cityRaw, 60),
+ 'meta.key': [],
+ 'meta.value': [],
+ importID,
+ created,
+ }
+
+ for (let i = 0; i < count; i++) {
+ yield { type: 'pageview', data }
+ }
+ }
+
+ if (rows.length < PAGE_SIZE) break
+ offset += PAGE_SIZE
+ }
+ }
+
+ private async *fetchCustomEvents(
+ client: BetaAnalyticsDataClient,
+ property: string,
+ dateRange: { start: string; end: string },
+ pid: string,
+ importID: number,
+ ): AsyncIterable<AnalyticsImportRow> {
+ let offset = 0
+
+ while (true) {
+ const [report] = await client.runReport({
+ property,
+ dateRanges: [{ startDate: dateRange.start, endDate: dateRange.end }],
+ dimensions: [
+ { name: 'dateHour' },
+ { name: 'eventName' },
+ { name: 'pagePath' },
+ { name: 'deviceCategory' },
+ { name: 'browser' },
+ { name: 'operatingSystem' },
+ { name: 'countryId' },
+ { name: 'hostName' },
+ ],
+ metrics: [{ name: 'eventCount' }],
+ limit: PAGE_SIZE,
+ offset,
+ keepEmptyRows: false,
+ })
+
+ const rows = report.rows || []
+ if (rows.length === 0) break
+
+ for (const row of rows) {
+ const dims = row.dimensionValues || []
+ const metrics = row.metricValues || []
+
+ const dateHour = dims[0]?.value || ''
+ const eventName = dims[1]?.value || ''
+ const pagePath = normalizeGA4Value(dims[2]?.value)
+ const deviceRaw = dims[3]?.value || null
+ const browserRaw = dims[4]?.value || null
+ const osRaw = dims[5]?.value || null
+ const countryId = normalizeGA4Value(dims[6]?.value)
+ const host = normalizeGA4Value(dims[7]?.value)
+
+ if (EXCLUDED_EVENTS.has(eventName)) continue
+
+ const count = parseInt(metrics[0]?.value || '0', 10)
+ if (count <= 0) continue
+
+ const created = this.parseGA4DateHour(dateHour)
+ if (!created) continue
+
+ const device = mapDevice(deviceRaw)
+ const browser = mapBrowser(browserRaw, device)
+ const os = mapOS(osRaw)
+ const cc = countryId && /^[A-Z]{2}$/.test(countryId) ? countryId : null
+
+ const sessionSeed = [
+ dateHour,
+ eventName,
+ pagePath,
+ deviceRaw,
+ browserRaw,
+ osRaw,
+ countryId,
+ ].join('|')
+ const psid = sessionIdToPsid(sessionSeed)
+
+ const data: Record<string, unknown> = {
+ psid,
+ profileId: null,
+ pid,
+ host: truncate(host, 253),
+ pg: truncate(pagePath, 2048),
+ event_name: truncate(eventName, 256),
+ dv: device,
+ br: truncate(browser, 30),
+ brv: null,
+ os: truncate(os, 25),
+ osv: null,
+ lc: null,
+ ref: null,
+ so: null,
+ me: null,
+ ca: null,
+ te: null,
+ co: null,
+ cc,
+ rg: null,
+ rgc: null,
+ ct: null,
+ 'meta.key': [],
+ 'meta.value': [],
+ importID,
+ created,
+ }
+
+ for (let i = 0; i < count; i++) {
+ yield { type: 'custom_event', data }
+ }
+ }
+
+ if (rows.length < PAGE_SIZE) break
+ offset += PAGE_SIZE
+ }
+ }
+
+ // GA4 dateHour format: "2024010115" β†’ "2024-01-01 15:00:00"
+ private parseGA4DateHour(raw: string): string | null {
+ if (!raw || raw.length < 10) return null
+ const year = raw.slice(0, 4)
+ const month = raw.slice(4, 6)
+ const day = raw.slice(6, 8)
+ const hour = raw.slice(8, 10)
+ return `${year}-${month}-${day} ${hour}:00:00`
+ }
+}
diff --git a/backend/apps/community/src/data-import/mappers/index.ts b/backend/apps/community/src/data-import/mappers/index.ts
index ce163a82d..406a0f1a5 100644
--- a/backend/apps/community/src/data-import/mappers/index.ts
+++ b/backend/apps/community/src/data-import/mappers/index.ts
@@ -3,11 +3,13 @@ import { UmamiMapper } from './umami.mapper'
import { SimpleAnalyticsMapper } from './simple-analytics.mapper'
import { FathomMapper } from './fathom.mapper'
import { PlausibleMapper } from './plausible.mapper'
+import { Ga4Mapper } from './ga4.mapper'
export const SUPPORTED_PROVIDERS = [
'umami',
'simple-analytics',
'fathom',
+ 'google-analytics',
'plausible',
] as const
@@ -15,6 +17,7 @@ const mappers: Record<string, ImportMapper> = {
umami: new UmamiMapper(),
'simple-analytics': new SimpleAnalyticsMapper(),
fathom: new FathomMapper(),
+ 'google-analytics': new Ga4Mapper(),
plausible: new PlausibleMapper(),
}
diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.ts
index 73ea8538b..cf19c69c8 100644
--- a/backend/apps/community/src/data-import/mappers/mapper.interface.ts
+++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts
@@ -12,11 +12,12 @@ export interface AnalyticsImportRow {
export interface ImportMapper {
readonly provider: string
- readonly expectedFileExtension: string
+ readonly expectedFileExtension: string | null
createRowStream(
- filePath: string,
+ filePath: string | null,
pid: string,
importID: number,
+ context?: Record<string, unknown>,
): AsyncIterable<AnalyticsImportRow>
}
diff --git a/docs/content/docs/selfhosting/configuring.mdx b/docs/content/docs/selfhosting/configuring.mdx
index 5538882a9..def624f3b 100644
--- a/docs/content/docs/selfhosting/configuring.mdx
+++ b/docs/content/docs/selfhosting/configuring.mdx
@@ -28,7 +28,7 @@ The `BASE_URL` must be publicly reachable from your users' browsers. Do not set
| Variable | Default | Description |
| ------------------------------------ | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `SECRET_KEY_BASE` | _auto-generated by `configure.sh`_ | Secret used for auth keys and login sessions. Must be set. |
-| `BASE_URL` | | Public URL of your Swetrix CE instance, e.g. `https://analytics.example.com`. Used by the API to build the OAuth redirect URI for the [Google Search Console integration](/selfhosting/google-search-console). Required to enable that integration. |
+| `BASE_URL` | | Public URL of your Swetrix CE instance, e.g. `https://analytics.example.com`. Used by the API to build OAuth redirect URIs for Google integrations. Required to enable [Google Search Console](/selfhosting/google-search-console) and [Google Analytics import](/selfhosting/google-analytics). |
| `CLIENT_IP_HEADER` | `null` | Defines what header should be used to get IP address (e.g. cf-connecting-ip, x-real-ip, etc.). If not set, Swetrix will use `x-forwarded-for` header. |
| `DISABLE_REGISTRATION` | `true` | When set to `true` - Swetrix will disable user registration, except for the initial user. |
| `IP_GEOLOCATION_DB_PATH` | `` | Path to the `.mmdb` file for IP geolocation database. By default Swetrix uses the IP to City Lite database by DB-IP, which provides country / region / city only. To populate the **Network intelligence** panel (ISP, organisation, usage type, connection type), point this at a paid MaxMind GeoIP2 ISP / Insights database or a paid DB-IP database that includes those fields. |
@@ -62,6 +62,18 @@ To let users connect their Google Search Console account from project settings (
If any of these are unset the SEO tab still shows up but the integrations card displays a "not configured" notice instead of the **Connect** button.
+### Google Analytics import
+
+To import data from Google Analytics 4, the API needs an OAuth client created in your own Google Cloud project. The full step-by-step guide lives in [Google Analytics import](/selfhosting/google-analytics); the relevant variables are:
+
+| Variable | Default | Description |
+| -------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `GOOGLE_GA4_CLIENT_ID` | | OAuth 2.0 Client ID issued by the Google Cloud console for your Swetrix instance. |
+| `GOOGLE_GA4_CLIENT_SECRET` | | OAuth 2.0 Client secret matching `GOOGLE_GA4_CLIENT_ID`. |
+| `BASE_URL` | | Public URL of your Swetrix CE instance. The API uses `${BASE_URL}/ga4-import-connected` as the OAuth redirect URI; it must match the URI registered in the Google Cloud console. |
+
+If any of these are unset, Google Analytics import requests fail with a "not configured" error.
+
### OIDC configuration
Swetrix Community Edition also supports OIDC authentication. To enable it, you need to set the following variables:
diff --git a/docs/content/docs/selfhosting/google-analytics.mdx b/docs/content/docs/selfhosting/google-analytics.mdx
new file mode 100644
index 000000000..08223a3d7
--- /dev/null
+++ b/docs/content/docs/selfhosting/google-analytics.mdx
@@ -0,0 +1,115 @@
+---
+title: Google Analytics import
+slug: /selfhosting/google-analytics
+---
+
+Swetrix Community Edition can import historical data from Google Analytics 4 (GA4). Unlike the cloud version, self-hosted instances need their own Google Cloud OAuth client because Google requires every OAuth app to be registered against a specific domain.
+
+This guide walks you through everything you need to do once.
+
+<Callout type="info">
+ Already have a Google Cloud project with an OAuth consent screen? Skip to [Issue an OAuth
+ client](#issue-an-oauth-client-and-secret).
+</Callout>
+
+## What you'll set up
+
+1. A Google Cloud project that owns the OAuth client.
+2. An OAuth consent screen for your Swetrix domain.
+3. A web-application OAuth client with the correct redirect URI.
+4. The Google Analytics Admin API enabled.
+5. The Google Analytics Data API enabled.
+6. Three environment variables on the Swetrix API container.
+
+## 1. Create a Google Cloud project
+
+Open the [Google Cloud console](https://console.cloud.google.com/) and either select an existing project or create a new one (we recommend a dedicated one for Swetrix).
+
+Make sure the project stays selected for every step below β€” Google sometimes silently switches you to a "default" project.
+
+## 2. Configure the OAuth consent screen
+
+In the left sidebar go to **APIs & Services** β†’ **OAuth consent screen**.
+
+- **User type:** **External** (the **Internal** option requires a Google Workspace organization).
+- **App name:** anything recognisable, e.g. `Swetrix`.
+- **User support email** and **Developer contact information:** your email.
+- **Authorized domains:** add the bare domain Swetrix is hosted on (no scheme, no path), e.g. `analytics.example.com`.
+- **Scopes:** add `https://www.googleapis.com/auth/analytics.readonly`.
+- **Test users:** add the Google account(s) that will import GA4 data from Swetrix. Google will warn you that the app is unverified; this is fine for self-hosted use.
+
+Save and continue through the wizard until the consent screen is registered.
+
+## 3. Issue an OAuth client and secret
+
+Still in **APIs & Services**, open **Credentials** and click **+ Create credentials** β†’ **OAuth client ID**.
+
+- **Application type:** **Web application**.
+- **Name:** `Swetrix CE GA4 Import` (or whatever you like).
+- **Authorized redirect URIs:** add **exactly one** entry:
+
+ ```txt
+ https://analytics.example.com/ga4-import-connected
+ ```
+
+ Replace `https://analytics.example.com` with the public URL you'll set as `BASE_URL` on the Swetrix API container. The path must be `/ga4-import-connected`.
+
+After you click **Create**, Google shows a **Client ID** and **Client secret**. Copy both β€” you'll need them in the next step.
+
+## 4. Enable the Google APIs
+
+Back in the Google Cloud console, go to **APIs & Services** β†’ **Library** and enable both APIs in the same project:
+
+- **Google Analytics Admin API** β€” used to list the GA4 properties the connected Google account can access.
+- **Google Analytics Data API** β€” used to read the analytics data during import.
+
+Without these APIs, authorisation or import requests will fail with Google API access errors.
+
+## 5. Configure the Swetrix API container
+
+Add the following variables to the `.env` file used by the `swetrix-api` container (or to your `docker-compose.yml`):
+
+```env
+# Public URL of your Swetrix CE instance. Must match the redirect URI you
+# registered in step 3 (without the /ga4-import-connected suffix).
+BASE_URL=https://analytics.example.com
+
+# OAuth client created in step 3
+GOOGLE_GA4_CLIENT_ID=1234567890-xxxxxxxxxxxxxxxxxxxxxxxx.apps.googleusercontent.com
+GOOGLE_GA4_CLIENT_SECRET=GOCSPX-xxxxxxxxxxxxxxxxxxxxxxxx
+```
+
+Restart the API container so it picks up the new environment:
+
+```bash
+docker compose up -d swetrix-api
+```
+
+You can confirm the variables are visible to the container with:
+
+```bash
+docker compose exec swetrix-api sh -c 'echo $GOOGLE_GA4_CLIENT_ID'
+```
+
+## 6. Start an import from Swetrix
+
+1. Open any project in Swetrix.
+2. Go to **Settings** β†’ **Data import**.
+3. Select **Google Analytics 4**.
+4. Click **Connect Google**. You'll be sent to Google's consent screen.
+5. Pick the Google account you added as a test user, click **Continue** through the unverified-app warnings, and approve the read-only Analytics scope.
+6. After Google redirects you back to Swetrix, select the GA4 property and start the import.
+
+Swetrix creates a background import job. Keep the project settings page open if you want to watch the status update.
+
+## Troubleshooting
+
+- **"Google Analytics import is not configured on this instance"** β€” one of `GOOGLE_GA4_CLIENT_ID`, `GOOGLE_GA4_CLIENT_SECRET`, or `BASE_URL` is missing. Check the API container env and restart it.
+- **`redirect_uri_mismatch`** during the Google consent flow β€” the redirect URI you registered in step 3 doesn't exactly match `${BASE_URL}/ga4-import-connected`. They must match character-for-character (scheme, hostname, port, trailing slash).
+- **Access denied while loading properties** β€” enable the Google Analytics Admin API and make sure the Google account has access to at least one GA4 property.
+- **Import fails after selecting a property** β€” enable the Google Analytics Data API in the same Google Cloud project and reconnect Google Analytics from Swetrix.
+- **`Access blocked: This app's request is invalid`** β€” your test user is not on the OAuth consent screen test-users list. Add them in step 2.
+
+## Rotating credentials
+
+If you ever rotate `GOOGLE_GA4_CLIENT_SECRET` or `SECRET_KEY_BASE`, existing GA4 connections become invalid. Users will need to reconnect Google Analytics before starting a new import.
diff --git a/docs/content/docs/selfhosting/meta.json b/docs/content/docs/selfhosting/meta.json
index a99a2f5aa..eea80cd4d 100644
--- a/docs/content/docs/selfhosting/meta.json
+++ b/docs/content/docs/selfhosting/meta.json
@@ -1,4 +1,4 @@
{
"title": "Self-hosting",
- "pages": ["how-to", "configuring", "google-search-console"]
+ "pages": ["how-to", "configuring", "google-search-console", "google-analytics"]
}
diff --git a/web/app/lib/models/Project.ts b/web/app/lib/models/Project.ts
index 5a832504d..184ddcc4e 100644
--- a/web/app/lib/models/Project.ts
+++ b/web/app/lib/models/Project.ts
@@ -1,4 +1,3 @@
-import { isSelfhosted } from '../constants'
import { Organisation, Role } from './Organisation'
interface OverallPeriodStats {
@@ -184,7 +183,7 @@ export type Provider =
export const IMPORT_PROVIDERS: Provider[] = [
'fathom',
- isSelfhosted ? null : 'google-analytics',
+ 'google-analytics',
'plausible',
'simple-analytics',
'umami',
diff --git a/web/app/routes/ga4-import-connected.tsx b/web/app/routes/ga4-import-connected.tsx
index cfb0a3b0b..fbf9e1434 100644
--- a/web/app/routes/ga4-import-connected.tsx
+++ b/web/app/routes/ga4-import-connected.tsx
@@ -6,7 +6,6 @@ import type { SitemapFunction } from 'remix-sitemap'
import { processGA4ImportTokenServer } from '~/api/api.server'
import { useAuthProxy } from '~/hooks/useAuthProxy'
-import { isSelfhosted } from '~/lib/constants'
import StatusPage from '~/ui/StatusPage'
import routes from '~/utils/routes'
import { getTitle } from '~/utils/seo'
@@ -28,10 +27,6 @@ export interface Ga4ImportConnectedLoaderData {
export async function loader({
request,
}: LoaderFunctionArgs): Promise<Ga4ImportConnectedLoaderData | Response> {
- if (isSelfhosted) {
- return redirect('/login', 302)
- }
-
const url = new URL(request.url)
const state = url.searchParams.get('state')