📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
import { ApiKeyCollection } from '../api-key/api-key-collection.decorator'
import { ApiKeyAccess } from '../api-key/api-key-access.decorator'
import _isEmpty from 'lodash/isEmpty'
import _isArray from 'lodash/isArray'
import _toNumber from 'lodash/toNumber'
import _includes from 'lodash/includes'
import _size from 'lodash/size'
import _map from 'lodash/map'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import dayjsTimezone from 'dayjs/plugin/timezone'
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger'
import {
Controller,
Body,
Query,
Param,
UseGuards,
UseInterceptors,
UsePipes,
ValidationPipe,
Get,
Post,
Patch,
Delete,
Headers,
HttpCode,
BadRequestException,
InternalServerErrorException,
UnprocessableEntityException,
Ip,
ForbiddenException,
Response,
Header,
HttpException,
HttpStatus,
StreamableFile,
} from '@nestjs/common'
import type { Response as ExpressResponse } from 'express'
import { OptionalJwtAccessTokenGuard } from '../auth/guards'
import { Auth, Public } from '../auth/decorators'
import {
AnalyticsService,
DataType,
getLowestPossibleTimeBucket,
} from './analytics.service'
import { normalizeFiltersToV1Json } from './v2/query/filters.translator'
import { VALID_PERIODS } from './decorators/validate-period.decorator'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
import { AuthenticationGuard } from '../auth/guards/authentication.guard'
import { PageviewsDto } from './dto/pageviews.dto'
import { EventsDto } from './dto/events.dto'
import { GetDataDto, ChartRenderMode } from './dto/getData.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetPagePropertyMetaDto } from './dto/get-page-property-meta.dto'
import { GetJourneysDto } from './dto/get-journeys.dto'
import { GetJourneySessionsDto } from './dto/get-journey-sessions.dto'
import { GetFunnelsDto } from './dto/getFunnels.dto'
import { GetFunnelSessionsDto } from './dto/get-funnel-sessions.dto'
import { AppLoggerService } from '../logger/logger.service'
import { AnalyticsReadGuard } from './protection/analytics-read.guard'
import { PublicProjectCacheInterceptor } from './protection/public-project-cache.interceptor'
import {
redis,
REDIS_USERS_COUNT_KEY,
REDIS_TRIALS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
ONLINE_VISITORS_WINDOW_MINUTES,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { checkRateLimit, getIPDetails, getIPFromHeaders } from '../common/utils'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
import {
DataDeletionDto,
DataDeletionPreviewDto,
} from './dto/data-deletion.dto'
import { GetVersionFiltersDto } from './dto/get-version-filters.dto'
import { IFunnel, IGetFunnel, IPageProperty, PerfMeasure } from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
import {
GetSessionReplayDto,
GetSessionReplaysDto,
SessionReplayExportStartDto,
SessionReplayChunkDto,
SessionReplayStartDto,
} from './dto/session-replay.dto'
import { SessionReplayExportService } from './session-replay-export.service'
import { GetProfilesDto } from './dto/get-profiles.dto'
import { GetProfileDto, GetProfileSessionsDto } from './dto/get-profile.dto'
import { ErrorDto } from './dto/error.dto'
import { GetErrorsDto } from './dto/get-errors.dto'
import { GetErrorDto } from './dto/get-error.dto'
import {
GetErrorOverviewDto,
GetErrorOverviewOptions,
} from './dto/get-error-overview.dto'
import { PatchStatusDto } from './dto/patch-status.dto'
import { OutboundDto } from './dto/outbound.dto'
import { eventTransformer } from './utils/transformers'
import { enrichTrafficSource } from './utils/clickIdSources'
import {
MAX_METRICS_IN_VIEW,
ProjectViewCustomEventDto,
} from '../project/dto/create-project-view.dto'
import { GetOverallStatsDto } from './dto/get-overall-stats.dto'
import { NoscriptDto } from './dto/noscript.dto'
import { LiveVisitorsDto } from './dto/live-visitors.dto'
import { GetHeartbeatStatsDto } from './dto/get-heartbeat-stats'
import { GetKeywordsDto } from './dto/get-keywords.dto'
import { GetBotStatsDto } from './dto/get-bot-stats.dto'
import { GSC_ALL_TIME_DAYS, GSCService } from '../project/gsc.service'
import { GetProfileIdDto, GetSessionIdDto } from './dto/get-id.dto'
import { IdentifyDto } from './dto/identify.dto'
import { ExperimentService } from '../experiment/experiment.service'
import {
ExperimentStatus,
ExposureTrigger,
} from '../experiment/entity/experiment.entity'
import { getExperimentVariant } from '../feature-flag/evaluation'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
const DEFAULT_MEASURE = 'median'
// Silent 200 response for bots
// https://github.com/Swetrix/swetrix/issues/371
const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
return (
perf.dns <= MAX_PERFORMANCE_VALUE &&
perf.tls <= MAX_PERFORMANCE_VALUE &&
perf.conn <= MAX_PERFORMANCE_VALUE &&
perf.response <= MAX_PERFORMANCE_VALUE &&
perf.render <= MAX_PERFORMANCE_VALUE &&
perf.dom_load <= MAX_PERFORMANCE_VALUE &&
perf.page_load <= MAX_PERFORMANCE_VALUE &&
perf.ttfb <= MAX_PERFORMANCE_VALUE &&
perf.dns >= 0 &&
perf.tls >= 0 &&
perf.conn >= 0 &&
perf.response >= 0 &&
perf.render >= 0 &&
perf.dom_load >= 0 &&
perf.page_load >= 0 &&
perf.ttfb >= 0
)
}
const getPIDsArray = (pids, pid) => {
const pidsEmpty = _isEmpty(pids)
const pidEmpty = _isEmpty(pid)
if (pidsEmpty && pidEmpty)
throw new BadRequestException(
"An array of Project ID's (pids) or a Project ID (pid) has to be provided",
)
else if (!pidsEmpty && !pidEmpty)
throw new BadRequestException(
"Please provide either an array of Project ID's (pids) or a Project ID (pid), but not both",
)
else if (!pidEmpty) {
pids = JSON.stringify([pid])
}
try {
pids = JSON.parse(pids)
} catch {
throw new UnprocessableEntityException(
"Cannot process the provided array of Project ID's",
)
}
if (!_isArray(pids)) {
throw new UnprocessableEntityException(
"An array of Project ID's has to be provided as a 'pids' param",
)
}
return pids
}
const getEIDsArray = (eids, eid) => {
const eidsEmpty = _isEmpty(eids)
const eidEmpty = _isEmpty(eid)
if (eidsEmpty && eidEmpty) {
throw new BadRequestException(
"An array of Error ID's (eids) or a Error ID (eid) has to be provided",
)
}
if (!eidsEmpty && !eidEmpty) {
throw new BadRequestException(
"Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",
)
}
if (!eidsEmpty) {
return eids
}
return [eid]
}
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
'base64',
)
const getPagePath = (page?: string | string[] | null) => {
const value = Array.isArray(page) ? page[0] : page
if (!value) return null
const path = value.trim().split('#')[0].split('?')[0]
if (!path.startsWith('/') || path.startsWith('//')) return null
return path || '/'
}
const getPageFromReferrer = (referrer?: string | string[] | null) => {
const value = Array.isArray(referrer) ? referrer[0] : referrer
if (!value) return null
try {
const url = new URL(value)
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
return getPagePath(url.pathname)
} catch {
return null
}
}
@ApiTags('Analytics')
@UseGuards(OptionalJwtAccessTokenGuard, AuthenticationGuard, AnalyticsReadGuard)
@UseInterceptors(PublicProjectCacheInterceptor)
@UsePipes(
new ValidationPipe({
transform: true,
}),
)
@Controller(['log', 'v1/log'])
export class AnalyticsController {
constructor(
private readonly analyticsService: AnalyticsService,
private readonly logger: AppLoggerService,
private readonly gscService: GSCService,
private readonly experimentService: ExperimentService,
private readonly sessionReplayExportService: SessionReplayExportService,
) {}
@ApiBearerAuth()
@Get()
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', ['events:read'])
async getData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
isCaptcha = false,
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
mode = ChartRenderMode.PERIODICAL,
metrics,
includeConcurrency,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}, metrics: ${metrics}`,
'GET /analytics',
)
let parsedMetrics: ProjectViewCustomEventDto[]
if (!isCaptcha) {
parsedMetrics = this.analyticsService.parseMetrics(metrics)
if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {
throw new BadRequestException(
`The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,
)
}
}
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll
let diff
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
)
if (period === 'all') {
const eventTypes = isCaptcha
? 'captcha'
: (['pageview', 'custom_event', 'error'] as const)
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
eventTypes,
)
diff = res.diff
newTimebucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
newTimebucket,
period,
safeTimezone,
diff,
)
const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(
filtersQuery,
customEVFilterApplied,
isCaptcha,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
let result: any | void
if (isCaptcha) {
result = await this.analyticsService.groupCaptchaByTimeBucket(
newTimebucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
mode,
period,
)
} else {
result = await this.analyticsService.groupByTimeBucket(
newTimebucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
appliedFilters,
mode,
includeConcurrency === 'true',
)
}
if (isCaptcha) {
return {
...result,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}
const customs = await this.analyticsService.getCustomEvents(
filtersQuery,
paramsData,
)
let properties: IPageProperty = {}
if (!customEVFilterApplied) {
properties = await this.analyticsService.getPageProperties(
filtersQuery,
paramsData,
)
}
let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResults>>
if (!_isEmpty(parsedMetrics)) {
meta = await this.analyticsService.getMetaResults(
pid,
parsedMetrics,
filtersQuery,
paramsData,
timezone,
period,
from,
to,
)
}
return {
...result,
customs,
properties,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
meta,
}
}
@Get('funnel')
@Auth(true, true)
@ApiKeyAccess('funnels:read', 'query', 'pid')
async getFunnel(
@Query() data: GetFunnelsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<IGetFunnel> {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
pages,
funnelId,
filters,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/funnel')
const pagesArr = await this.analyticsService.getPagesArray(
pages,
funnelId,
pid,
)
let diff
if (period === 'all') {
const [pageviewRes, customEventRes] = await Promise.all([
this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
),
])
diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters || '[]',
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
let funnel: IFunnel[] = []
let totalPageviews: number = 0
let timeToConvert: IGetFunnel['timeToConvert']
let stepDetails = {
countries: {},
devices: {},
browsers: {},
sources: {},
campaigns: {},
pages: {},
profileTypes: {},
} as Record<string, Record<number, Record<string, number>>>
const promises = [
(async () => {
funnel = await this.analyticsService.getFunnel(
pagesArr,
params,
filtersQuery,
)
})(),
(async () => {
totalPageviews = await this.analyticsService.getTotalPageviews(
pid,
groupFrom,
groupTo,
filtersQuery,
filtersParams,
)
})(),
(async () => {
try {
stepDetails = await this.analyticsService.getFunnelStepDetails(
pagesArr,
params,
filtersQuery,
)
} catch (e) {
this.logger.error(e, 'GET /analytics/funnel - getFunnelStepDetails')
}
})(),
(async () => {
try {
timeToConvert = await this.analyticsService.getFunnelTimeToConvert(
pagesArr,
params,
filtersQuery,
)
} catch (e) {
this.logger.error(e, 'GET /analytics/funnel - getFunnelTimeToConvert')
}
})(),
]
await Promise.all(promises)
for (let i = 0; i < funnel.length; i++) {
funnel[i].topCountries = stepDetails.countries[i + 1] || {}
funnel[i].topSources = stepDetails.sources[i + 1] || {}
funnel[i].breakdowns = {
countries: stepDetails.countries[i + 1] || {},
devices: stepDetails.devices[i + 1] || {},
browsers: stepDetails.browsers[i + 1] || {},
sources: stepDetails.sources[i + 1] || {},
campaigns: stepDetails.campaigns[i + 1] || {},
pages: stepDetails.pages[i + 1] || {},
profileTypes: stepDetails.profileTypes[i + 1] || {},
}
}
return { funnel, totalPageviews, timeToConvert }
}
@Get('funnel-sessions')
@Auth(true, true)
@ApiKeyAccess('funnels:read', 'query', 'pid', [
'analytics:read',
'events:read',
'errors:read',
])
async getFunnelSessions(
@Query() data: GetFunnelSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
pages,
funnelId,
step,
filters,
dropoff,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const pagesArr = await this.analyticsService.getPagesArray(
pages,
funnelId,
pid,
)
if (step < 1 || step > pagesArr.length) {
throw new BadRequestException(
'Step must be between 1 and the number of funnel steps',
)
}
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, step: ${step}, take: ${take}, skip: ${skip}`,
'GET /analytics/funnel-sessions',
)
let diff
if (period === 'all') {
const [pageviewRes, customEventRes] = await Promise.all([
this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
),
])
diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters || '[]',
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
const isDropoff = dropoff === true || dropoff === 'true'
const sessions = await this.analyticsService.getFunnelSessionsList(
pagesArr,
params,
safeTimezone,
step,
take,
skip,
filtersQuery,
isDropoff,
)
return { sessions, take, skip }
}
@Get('meta')
@Auth(true, true)
@ApiKeyAccess('events:read', 'query', 'pid')
async getCustomEventMetadata(
@Query() data: GetCustomEventMetadata,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {
const { pid } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}`, 'GET /analytics/meta')
return this.analyticsService.getCustomEventMetadata(data)
}
@Get('property')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPagePropertyMetadata(
@Query() data: GetPagePropertyMetaDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {
const { pid } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}`, 'GET /analytics/property')
return this.analyticsService.getPagePropertyMeta(data)
}
@Get('filters')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}, type: ${type}`, 'GET /analytics/filters')
return this.analyticsService.getFilters(pid, type)
}
@Get('errors-filters')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrorsFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, type: ${type}`,
'GET /analytics/errors-filters',
)
return this.analyticsService.getErrorsFilters(pid, type)
}
@Get('filters/versions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getVersionFilters(
@Query() data: GetVersionFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type, column } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, type: ${type}, column: ${column}`,
'GET /analytics/filters/versions',
)
return this.analyticsService.getVersionFilters(pid, type, column)
}
@Get('chart')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getChartData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
mode = ChartRenderMode.PERIODICAL,
includeConcurrency,
} = data
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart')
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const result = await this.analyticsService.groupChartByTimeBucket(
timeBucket,
groupFrom,
groupTo,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
mode,
includeConcurrency === 'true',
)
return { ...result, appliedFilters }
}
@Get('performance')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerfData(
@Query() data: GetDataDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
measure = DEFAULT_MEASURE,
} = data
this.analyticsService.checkIfPerfMeasureIsValid(measure)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}, measure: ${measure}`,
'GET /analytics/performance',
)
let newTimeBucket = timeBucket
let allowedTumebucketForPeriodAll
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'performance',
)
diff = res.diff
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const result = await this.analyticsService.groupPerfByTimeBucket(
newTimeBucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
measure,
)
return {
...result,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}
@Get('performance/chart')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerfChartData(
@Query() data: GetDataDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
measure = DEFAULT_MEASURE,
} = data
this.analyticsService.checkIfPerfMeasureIsValid(measure)
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}, measure: ${measure}`,
'GET /analytics/performance/chart',
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const chart = await this.analyticsService.getPerfChartData(
timeBucket,
groupFrom,
groupTo,
filtersQuery,
paramsData,
safeTimezone,
measure,
)
return { chart, appliedFilters }
}
@Get('captcha')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getCaptchaData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
) {
return this.getData(data, uid, {}, true)
}
@Get('journeys')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getJourneys(
@Query() data: GetJourneysDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const steps = this.analyticsService.getSafeNumber(data.steps, 3)
const journeys = this.analyticsService.getSafeNumber(data.journeys, 20)
this.logger.log(
`pid: ${pid}, period: ${period}, steps: ${steps}, journeys: ${journeys}`,
'GET /analytics/journeys',
)
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'pageview',
)
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
true,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
let nodeDetails = []
let linkDetails = []
const [flow] = await Promise.all([
this.analyticsService.getJourneys(params, filtersQuery, steps, journeys),
(async () => {
try {
const details = await this.analyticsService.getJourneyNodeDetails(
params,
filtersQuery,
steps,
journeys,
)
nodeDetails = details.nodes
linkDetails = details.links
} catch (e) {
this.logger.error(
e,
'GET /analytics/journeys - getJourneyNodeDetails',
)
}
})(),
])
return { ...flow, nodeDetails, linkDetails, appliedFilters }
}
@Get('journey-sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'analytics:read',
'events:read',
'errors:read',
])
async getJourneySessions(
@Query() data: GetJourneySessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
step,
page,
filters,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, step: ${step}, take: ${take}, skip: ${skip}`,
'GET /analytics/journey-sessions',
)
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'pageview',
)
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
const sessions = await this.analyticsService.getJourneySessionsList(
params,
safeTimezone,
step,
page,
take,
skip,
filtersQuery,
)
return { sessions, take, skip }
}
@Get('keywords')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getKeywords(
@Query() data: GetKeywordsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/keywords')
const diff = period === 'all' ? GSC_ALL_TIME_DAYS : undefined
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const keywords = await this.gscService.getKeywords(pid, groupFrom, groupTo)
return { keywords }
}
@Get('birdseye')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
// returns overall short statistics per project
async getOverallStats(
@Query() data: GetOverallStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pids,
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
timeBucket,
filters,
includeChart,
} = data
const pidsArray = getPIDsArray(pids, pid)
this.logger.log(`pidsArray: ${pidsArray}`, 'GET /analytics/birdseye')
const validPids = []
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(currentPID)
validPids.push(currentPID)
})
try {
await Promise.allSettled(validationPromises)
} catch {
//
}
if (_isEmpty(validPids)) {
throw new HttpException(
'The data could not be loaded for the selected projects. It is possible that the projects are not accessible to you or the account owner has been suspended.',
HttpStatus.PAYMENT_REQUIRED,
)
}
return this.analyticsService.getAnalyticsSummary(
validPids,
timeBucket,
period,
from,
to,
timezone,
filters,
includeChart,
)
}
@Get('performance/birdseye')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerformanceOverallStats(
@Query() data: GetOverallStatsDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pids,
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
filters,
} = data
let { measure = DEFAULT_MEASURE } = data
const pidsArray = getPIDsArray(pids, pid)
this.analyticsService.checkIfPerfMeasureIsValid(measure)
if (measure === 'quantiles') {
measure = DEFAULT_MEASURE
}
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(currentPID)
})
await Promise.all(validationPromises)
this.logger.log(
`pidsArray: ${pidsArray}, measure: ${measure}`,
'GET /analytics/performance/birdseye',
)
return this.analyticsService.getPerformanceSummary(
pidsArray,
period,
from,
to,
timezone,
filters,
measure,
)
}
@Public()
@Get('generalStats')
async getGeneralStats(): Promise<object> {
const exists = await redis.exists(
REDIS_USERS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
REDIS_TRIALS_COUNT_KEY,
)
if (exists) {
const users = _toNumber(await redis.get(REDIS_USERS_COUNT_KEY))
const trials = _toNumber(await redis.get(REDIS_TRIALS_COUNT_KEY))
const projects = _toNumber(await redis.get(REDIS_PROJECTS_COUNT_KEY))
const events = _toNumber(await redis.get(REDIS_EVENTS_COUNT_KEY))
return { users, trials, projects, events }
}
return this.analyticsService.getGeneralStats()
}
@Get('hb')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getHeartBeatStats(
@Query() data: GetHeartbeatStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pids, pid } = data
const pidsArray = getPIDsArray(pids, pid)
this.logger.log(`pidsArray: ${pidsArray}`, 'GET /analytics/hb')
const validPids = []
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(currentPID)
validPids.push(currentPID)
})
try {
await Promise.allSettled(validationPromises)
} catch {
//
}
if (_isEmpty(validPids)) {
throw new HttpException(
'The data could not be loaded for the selected projects. It is possible that the projects are not accessible to you or the account owner has been suspended.',
HttpStatus.PAYMENT_REQUIRED,
)
}
const result = {}
const keyCountPromises = _map(validPids, async (currentPID) => {
result[currentPID] =
await this.analyticsService.getOnlineUserCount(currentPID)
})
await Promise.all(keyCountPromises).catch((reason) => {
this.logger.error(`[GET /analytics/hb] ${reason}`)
throw new InternalServerErrorException(
'An error occured while calculating heartbeat statistics',
)
})
return result
}
@Get('live-visitors')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getLiveVisitors(
@Query() queryParams: LiveVisitorsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pid } = queryParams
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}`, 'GET /analytics/live-visitors')
const since = dayjs
.utc()
.subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')
.format('YYYY-MM-DD HH:mm:ss')
const query = `
SELECT
dv,
br,
os,
cc,
psidCasted AS psid
FROM (
SELECT
any(dv) AS dv,
any(br) AS br,
any(os) AS os,
any(cc) AS cc,
toString(psid) AS psidCasted
FROM events
WHERE
pid = {pid:FixedString(12)}
AND type IN ('pageview', 'custom_event')
AND created >= {since:DateTime}
AND psid IS NOT NULL
AND psid != 0
GROUP BY psid
)
`
const { data } = await clickhouse
.query({
query,
query_params: {
pid,
since,
},
})
.then((resultSet) => resultSet.json())
return data
}
@Get('bot-stats')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getBotStats(
@Query() data: GetBotStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period = '30d' } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}`,
'GET /analytics/bot-stats',
)
return this.analyticsService.getBotStats(pid, period)
}
@Post('session-replay/start')
@ApiKeyCollection()
@ApiKeyAccess('replays:write', 'body', 'pid')
@Public()
async startSessionReplay(
@Body() replayDTO: SessionReplayStartDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'session-replay-start', 120, 60)
await checkRateLimit(replayDTO.pid, 'session-replay-start', 2000, 60)
const botResult = await this.analyticsService.checkBot(
replayDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
replayDTO.pg,
'session_replay',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(replayDTO, origin, ip)
const { country } = getIPDetails(ip, replayDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${replayDTO.pid}, replayId: ${replayDTO.replayId}`,
'POST /analytics/session-replay/start',
)
return this.analyticsService.startSessionReplay(
project,
replayDTO.pid,
replayDTO.replayId,
replayDTO.privacy,
userAgent,
ip,
replayDTO.profileId,
)
}
@Post('session-replay/chunk')
@ApiKeyCollection()
@ApiKeyAccess('replays:write', 'body', 'pid')
@Public()
async uploadSessionReplayChunk(
@Body() replayDTO: SessionReplayChunkDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'session-replay-chunk', 600, 60)
await checkRateLimit(replayDTO.pid, 'session-replay-chunk', 5000, 60)
const botResult = await this.analyticsService.checkBot(
replayDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
replayDTO.pg,
'session_replay',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(replayDTO, origin, ip)
const { country } = getIPDetails(ip, replayDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${replayDTO.pid}, replayId: ${replayDTO.replayId}, chunkIndex: ${replayDTO.chunkIndex}`,
'POST /analytics/session-replay/chunk',
)
return this.analyticsService.storeSessionReplayChunk(
project,
replayDTO.pid,
replayDTO.replayId,
replayDTO.privacy,
replayDTO.chunkIndex,
replayDTO.events,
userAgent,
ip,
)
}
@Post('error')
@ApiKeyCollection()
@ApiKeyAccess('errors:write', 'body', 'pid')
@Public()
async logError(@Body() errorDTO: ErrorDto, @Headers() headers, @Ip() reqIP) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
errorDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
errorDTO.pg,
'error',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(errorDTO, origin, ip)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
errorDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
errorDTO.pid,
userAgent,
ip,
errorDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
errorDTO.pid,
profileId,
isNewSession,
)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, errorDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${errorDTO.pid}, pg: ${errorDTO.pg}, name: ${errorDTO.name}, message: ${errorDTO.message}`,
'POST /analytics/error',
)
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const { name, message, lineno, colno, filename, stackTrace, meta } =
errorDTO
const transformed = eventTransformer({
type: 'error',
psid,
sid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
pid: errorDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: errorDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: errorDTO.lc,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
name,
message,
lineno,
colno,
filename,
stackTrace,
meta,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
} catch (reason) {
this.logger.error(reason)
throw new InternalServerErrorException(
'Error occured while saving the custom event',
)
}
return {}
}
// Update error(s) status
@Patch('error-status')
@Auth(true, true)
@ApiKeyAccess('errors:write', 'body', 'pid')
async patchStatus(
@Body() statusDTO: PatchStatusDto,
@CurrentUserId() uid: string,
@Headers() headers,
@Ip() reqIP,
) {
const { pid, eid, eids: unprocessedEids, status } = statusDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'error-status', 100, 1800)
await this.analyticsService.checkManageAccess(pid, uid)
await this.analyticsService.checkBillingAccess(pid)
const eids = getEIDsArray(unprocessedEids, eid)
await this.analyticsService.validateEIDs(eids, pid)
return this.analyticsService.updateEIDStatus(eids, status, pid)
}
@Post('custom')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async logCustom(
@Body() eventsDTO: EventsDto,
@Headers() headers,
@Ip() reqIP,
) {
return this.logInteraction(eventsDTO, headers, reqIP)
}
@Post('outbound')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async logOutbound(
@Body() outboundDTO: OutboundDto,
@Headers() headers,
@Ip() reqIP,
) {
return this.logInteraction(
{ ...outboundDTO, ev: '', unique: false },
headers,
reqIP,
outboundDTO.url,
outboundDTO.title,
)
}
private async logInteraction(
eventsDTO: EventsDto,
headers,
reqIP,
outboundUrl?: string,
outboundTitle?: string | null,
) {
const isOutbound = outboundUrl !== undefined
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
eventsDTO.pid,
userAgent,
headers,
ip,
eventsDTO.ref || headers.referer || headers.referrer,
eventsDTO.pg,
'custom',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(eventsDTO, origin, ip)
if (!isOutbound && eventsDTO.unique) {
const [unique] = await this.analyticsService.generateAndStoreSessionId(
`${eventsDTO.pid}-${eventsDTO.ev}`,
userAgent,
ip,
)
if (!unique) {
throw new ForbiddenException(
'The unique option provided, while the custom event have already been created for this session',
)
}
}
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, eventsDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${eventsDTO.pid}, event: ${isOutbound ? outboundUrl : eventsDTO.ev}, pg: ${eventsDTO.pg}`,
'POST /analytics/custom',
)
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
eventsDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
eventsDTO.pid,
userAgent,
ip,
eventsDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
eventsDTO.pid,
profileId,
isNewSession,
)
enrichTrafficSource(eventsDTO)
const transformed = eventTransformer({
...(isOutbound
? { type: 'outbound' as const, url: outboundUrl, title: outboundTitle }
: { type: 'custom_event' as const, ev: eventsDTO.ev }),
psid,
sid,
profileId,
pid: eventsDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: eventsDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: eventsDTO.lc,
ref: eventsDTO.ref,
so: eventsDTO.so,
me: eventsDTO.me,
ca: eventsDTO.ca,
te: eventsDTO.te,
co: eventsDTO.co,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: eventsDTO.meta,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
if (!isOutbound)
void this.trackCustomEventExperimentExposures(
eventsDTO.pid,
eventsDTO.ev,
profileId,
transformed.created,
)
} catch (reason) {
this.logger.error(reason)
throw new InternalServerErrorException(
'Error occurred while saving the event',
)
}
return {}
}
private async trackCustomEventExperimentExposures(
pid: string,
eventName: string,
profileId: string,
created: string,
) {
try {
const experiments = await this.experimentService.find({
where: {
project: { id: pid },
status: ExperimentStatus.RUNNING,
exposureTrigger: ExposureTrigger.CUSTOM_EVENT,
customEventName: eventName,
},
relations: ['variants'],
})
if (_isEmpty(experiments)) {
return
}
const exposures = []
for (const experiment of experiments) {
if (!experiment.variants || experiment.variants.length === 0) {
continue
}
const sortedVariants = [...experiment.variants].sort((a, b) =>
a.key.localeCompare(b.key),
)
const variantKey = getExperimentVariant(
experiment.id,
sortedVariants.map((variant) => ({
key: variant.key,
rolloutPercentage: variant.rolloutPercentage,
})),
profileId,
)
if (!variantKey) {
continue
}
exposures.push({
pid,
experimentId: experiment.id,
variantKey,
profileId,
created,
})
}
if (_isEmpty(exposures)) {
return
}
clickhouse
.insert({
table: 'experiment_exposures',
values: exposures,
format: 'JSONEachRow',
clickhouse_settings: { async_insert: 1 },
})
.catch((reason) => {
this.logger.warn(
{ reason },
'Failed to async insert custom event experiment exposures',
)
})
} catch (reason) {
this.logger.warn(
{ reason },
'Failed to track custom event experiment exposures',
)
}
}
@Post('hb')
@Auth(true, true)
@ApiKeyAccess('events:write', 'body', 'pid')
async heartbeat(
@Body() logDTO: PageviewsDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const { pid } = logDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
logDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
logDTO.pg,
'heartbeat',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
if (!exists || !sid) {
throw new ForbiddenException(
'The heartbeat was not saved because there is no session for this request. Please, send a pageview or custom event request first to initialise the session.',
)
}
const profileId = await this.analyticsService.generateProfileId(
pid,
userAgent,
ip,
logDTO.profileId,
)
await this.analyticsService.extendSessionTTL(psid)
await this.analyticsService.recordSessionActivity(sid, psid, pid, profileId)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'POST /analytics/hb')
return {}
}
// Log pageview event
@Post()
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async log(@Body() logDTO: PageviewsDto, @Headers() headers, @Ip() reqIP) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
logDTO.pid,
userAgent,
headers,
ip,
logDTO.ref || headers.referer || headers.referrer,
logDTO.pg,
'pageview',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(logDTO, origin, ip)
const [unique, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
logDTO.pid,
userAgent,
ip,
logDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
logDTO.pid,
profileId,
unique,
)
if (!unique && logDTO.unique) {
throw new ForbiddenException(
'The event was not saved because it was not unique while unique only param is provided',
)
}
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, logDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(`pid: ${logDTO.pid}, pg: ${logDTO.pg}`, 'POST /analytics')
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
enrichTrafficSource(logDTO)
const transformed = eventTransformer({
type: 'pageview',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
title: logDTO.title,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: logDTO.lc,
ref: logDTO.ref,
so: logDTO.so,
me: logDTO.me,
ca: logDTO.ca,
te: logDTO.te,
co: logDTO.co,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: logDTO.meta,
})
let perfTransformed = null
if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {
const {
dns,
tls,
conn,
response,
render,
dom_load: domLoad,
page_load: pageLoad,
ttfb,
} = logDTO.perf
perfTransformed = eventTransformer({
type: 'performance',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
dns,
tls,
conn,
response,
render,
domLoad,
pageLoad,
ttfb,
})
}
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
if (!_isEmpty(perfTransformed)) {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [perfTransformed],
clickhouse_settings: { async_insert: 1 },
})
}
} catch (reason) {
this.logger.error(reason)
throw new InternalServerErrorException(
'Error occured while saving the log data',
)
}
return {}
}
// Fallback for logging pageviews for users with JavaScript disabled
// Returns 1x1 transparent gif
@Get('noscript')
@Header('Cross-Origin-Resource-Policy', 'cross-origin')
@Public()
async noscript(
@Query() data: NoscriptDto,
@Headers() headers,
@Ip() reqIP,
@Response() res,
) {
const { 'user-agent': userAgent, origin } = headers
const { pid } = data
const referrer = headers.referer || headers.referrer
const pg =
getPagePath(data.pg) || getPageFromReferrer(referrer) || undefined
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
pid,
userAgent,
headers,
ip,
referrer,
pg ?? null,
'noscript',
)
if (botResult.isBot) {
res.writeHead(200, { 'Content-Type': 'image/gif' })
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
const logDTO: PageviewsDto = { pid, pg }
const project = await this.analyticsService.validate(logDTO, origin, ip)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
logDTO.pid,
userAgent,
ip,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
logDTO.pid,
profileId,
isNewSession,
)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(`pid: ${pid}, pg: ${pg}`, 'GET /analytics/noscript')
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const transformed = eventTransformer({
type: 'pageview',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: null,
ref: null,
so: null,
me: null,
ca: null,
te: null,
co: null,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: null,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
} catch (reason) {
this.logger.error(reason)
}
res.writeHead(200, { 'Content-Type': 'image/gif' })
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
@Get('sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
'replays:read',
'revenue:read',
])
async getSessions(
@Query() data: GetSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
const sessionEvent = data.sessionEvent || 'traffic'
const sessionDataType =
sessionEvent === 'performance'
? DataType.PERFORMANCE
: sessionEvent === 'error'
? DataType.ERRORS
: DataType.ANALYTICS
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/sessions',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
sessionDataType,
sessionEvent !== 'traffic',
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
sessionEvent === 'traffic'
? (['pageview', 'custom_event', 'error'] as const)
: sessionEvent,
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const sessions = await this.analyticsService.getSessionsList(
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
sessionEvent,
)
return { sessions, appliedFilters, take, skip }
}
@Get('session-replays')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'query', 'pid')
async getSessionReplays(
@Query() data: GetSessionReplaysDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of session replays to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/session-replays',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
['pageview', 'custom_event', 'error'] as const,
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const replays = await this.analyticsService.getSessionReplaysList(
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
)
return { replays, appliedFilters, take, skip }
}
@Get('errors')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrors(
@Query() data: GetErrorsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
options,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of errors to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/errors',
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ERRORS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const errors = await this.analyticsService.getErrorsList(
options,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
)
return { errors, appliedFilters, take, skip }
}
@Get('get-error')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getError(
@Query() data: GetErrorDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
timezone = DEFAULT_TIMEZONE,
eid,
period,
from,
to,
timeBucket,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, eid: ${eid}, period: ${period}`,
'GET /analytics/get-error',
)
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const result = await this.analyticsService.getErrorDetails(
pid,
eid,
safeTimezone,
groupFromUTC,
groupToUTC,
newTimeBucket,
)
return result
}
@Get('error-overview')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrorOverview(
@Query() data: GetErrorOverviewDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
timeBucket,
options,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}`,
'GET /analytics/error-overview',
)
let parsedOptions: GetErrorOverviewOptions = {}
if (typeof options === 'string') {
try {
parsedOptions = JSON.parse(options)
} catch {
// Ignore parse errors
}
} else {
parsedOptions = options || {}
}
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters,
DataType.ERRORS,
true,
)
const [sessionFiltersQuery, sessionFiltersParams] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
return this.analyticsService.getErrorOverview(
pid,
filtersQuery,
paramsData,
safeTimezone,
groupFromUTC,
groupToUTC,
newTimeBucket,
parsedOptions.showResolved || false,
sessionFiltersQuery,
sessionFiltersParams,
)
}
@Get('error-sessions')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid', ['analytics:read'])
async getErrorSessions(
@Query() data: GetErrorDto & { take?: number; skip?: number },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
eid,
period,
from,
to,
timeBucket,
filters,
timezone = DEFAULT_TIMEZONE,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 10)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 50) {
throw new BadRequestException(
'The maximum number of sessions to return is 50',
)
}
this.logger.log(
`pid: ${pid}, eid: ${eid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/error-sessions',
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters,
DataType.ERRORS,
true,
)
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
return this.analyticsService.getErrorAffectedSessions(
pid,
eid,
groupFromUTC,
groupToUTC,
take,
skip,
filtersQuery,
filtersParams,
)
}
@Get('session')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
'replays:read',
'revenue:read',
])
async getSession(
@Query() data: GetSessionDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, psid, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'GET /analytics/session')
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const result = await this.analyticsService.getSessionDetails(
pid,
psid,
safeTimezone,
)
return result
}
@Get('session-replay')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'query', 'pid')
async getSessionReplay(
@Query() data: GetSessionReplayDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'GET /analytics/session-replay',
)
return this.analyticsService.getSessionReplay(pid, psid, replayId)
}
@Delete('session-replay')
@Auth(true, true)
@ApiKeyAccess('replays:write', 'query', 'pid')
async deleteSessionReplay(
@Query() data: GetSessionReplayDto,
@CurrentUserId() uid: string,
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkManageAccess(pid, uid)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'DELETE /analytics/session-replay',
)
return this.analyticsService.deleteSessionReplay(pid, psid, replayId)
}
@Post('data-deletion/preview')
@HttpCode(200)
@Auth()
async getDataDeletionPreview(
@Body() body: DataDeletionPreviewDto,
@CurrentUserId() uid: string,
) {
await this.analyticsService.checkManageAccess(body.pid, uid)
await checkRateLimit(uid, 'data-deletion-preview-user', 30, 60)
await checkRateLimit(body.pid, 'data-deletion-preview-project', 60, 60)
return this.analyticsService.getDataDeletionPreview(
body.pid,
normalizeFiltersToV1Json(body.filters, 'traffic'),
body.from || null,
body.to || null,
)
}
@Post('data-deletion')
@HttpCode(200)
@Auth()
async deleteData(
@Body() body: DataDeletionDto,
@CurrentUserId() uid: string,
) {
await this.analyticsService.checkManageAccess(body.pid, uid)
await checkRateLimit(uid, 'data-deletion-user', 5, 60)
await checkRateLimit(body.pid, 'data-deletion-project', 10, 60)
this.logger.log(
{ uid, pid: body.pid, types: body.types },
'POST /log/data-deletion',
)
await this.analyticsService.deleteData(
body.pid,
normalizeFiltersToV1Json(body.filters, 'traffic'),
body.from || null,
body.to || null,
body.types,
)
return { deleted: true }
}
@Post('session-replay/export')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'body', 'pid')
async startSessionReplayExport(
@Body() data: SessionReplayExportStartDto,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string; 'user-agent'?: string },
@Ip() requestIp: string,
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const ip = getIPFromHeaders(headers) || requestIp || ''
await checkRateLimit(uid || ip || pid, 'session-replay-export-user', 5, 60)
await checkRateLimit(pid, 'session-replay-export-project', 20, 60)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'POST /analytics/session-replay/export',
)
return this.sessionReplayExportService.startExport(pid, psid, replayId)
}
@Get('session-replay/export/:exportId')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'export', 'exportId')
async getSessionReplayExport(
@Param('exportId') exportId: string,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string },
) {
const exportState =
await this.sessionReplayExportService.assertExportAccess(exportId)
await this.analyticsService.checkProjectAccess(
exportState.pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(exportState.pid)
return this.sessionReplayExportService.getExport(exportId)
}
@Get('session-replay/export/:exportId/download')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'export', 'exportId')
async downloadSessionReplayExport(
@Param('exportId') exportId: string,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string },
@Response({ passthrough: true }) response: ExpressResponse,
): Promise<StreamableFile> {
const exportState =
await this.sessionReplayExportService.assertExportAccess(exportId)
await this.analyticsService.checkProjectAccess(
exportState.pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(exportState.pid)
return this.sessionReplayExportService.downloadExport(exportId, response)
}
@Get('profiles')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
])
async getProfiles(
@Query() data: GetProfilesDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
profileType = 'all',
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of profiles to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/profiles',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const profiles = await this.analyticsService.getProfilesList(
pid,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
profileType,
customEVFilterApplied,
)
return { profiles, appliedFilters, take, skip }
}
@Get('profile')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
'revenue:read',
])
async getProfile(
@Query() data: GetProfileDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, profileId, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, profileId: ${profileId}`,
'GET /analytics/profile',
)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const [details, topPages, activityCalendar] = await Promise.all([
this.analyticsService.getProfileDetails(pid, profileId, safeTimezone),
this.analyticsService.getProfileTopPages(pid, profileId),
this.analyticsService.getProfileActivityCalendar(pid, profileId),
])
return {
...details,
topPages,
activityCalendar,
}
}
@Get('profile/sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
])
async getProfileSessions(
@Query() data: GetProfileSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
profileId,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, profileId: ${profileId}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/profile/sessions',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const sessions = await this.analyticsService.getProfileSessionsList(
pid,
profileId,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
)
return { sessions, appliedFilters, take, skip }
}
@Get('custom-events')
@Auth(true, true)
@ApiKeyAccess('events:read', 'query', 'pid')
async getCustomEvents(
@Query() data: GetCustomEventsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
customEvents,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
await this.analyticsService.checkBillingAccess(pid)
this.logger.log(
`pid: ${pid}, period: ${period}, customEvents: ${customEvents}`,
'GET /analytics/custom-events',
)
let newTimeBucket = timeBucket
let diff
let timeBucketForAllTime
if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
)
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
diff = res.diff
timeBucketForAllTime = res.timeBucket
}
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
// customEvents comes as a JSON.stringified array from the frontend
let customEventsList: string[] = []
try {
const parsed = JSON.parse(customEvents || '[]')
customEventsList = Array.isArray(parsed) ? parsed : []
} catch {
customEventsList = []
}
if (customEventsList.length === 0) {
return { chart: {}, appliedFilters, timeBucket: timeBucketForAllTime }
}
const result: any = await this.analyticsService.groupCustomEVByTimeBucket(
newTimeBucket,
groupFrom,
groupTo,
filtersQuery,
paramsData,
safeTimezone,
customEventsList,
)
return { ...result, appliedFilters, timeBucket: timeBucketForAllTime }
}
/**
* Links the visitor's current anonymous profile to an identified profile
* derived from the user ID supplied by the site (e.g. after log in). Events
* previously recorded for the anonymous profile get attributed to the
* identified profile at query time; the tracker stamps all subsequent
* events with the supplied profileId directly.
*
* Optional traits (email, plan, ...) are stored against the identified
* profile and shown on its dashboard page.
*/
@Post('identify')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async identify(
@Body() dto: IdentifyDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ profileId: string } | typeof BOT_RESPONSE> {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'identify', 120, 60)
await checkRateLimit(dto.pid, 'identify', 2000, 60)
const botResult = await this.analyticsService.checkBot(
dto.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
null,
'identify',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const profileId = this.analyticsService.validateUserSuppliedProfileId(
dto.profileId,
)
await this.analyticsService.validate(dto, origin, ip)
const userProfileId = await this.analyticsService.generateProfileId(
dto.pid,
userAgent,
ip,
profileId,
)
if (!_isEmpty(dto.traits)) {
await this.analyticsService.saveProfileTraits(
dto.pid,
userProfileId,
dto.traits,
)
}
const anonProfileId = await this.analyticsService.generateProfileId(
dto.pid,
userAgent,
ip,
)
const linked = await this.analyticsService.linkProfiles(
dto.pid,
anonProfileId,
userProfileId,
)
// Flip the visitor's current session (if any) to the identified profile.
// Skipped when the anonymous profile is already linked to a different
// identified profile (e.g. a second account on a shared device) - the
// session stays with the current identity until new events re-stamp it.
if (linked) {
const { exists, psid, sid } = await this.analyticsService.getSessionId(
dto.pid,
userAgent,
ip,
)
if (exists && sid) {
await this.analyticsService.recordSessionActivity(
sid,
psid,
dto.pid,
userProfileId,
)
}
}
this.logger.log(
`pid: ${dto.pid}, profileId: ${userProfileId}`,
'POST /analytics/identify',
)
return { profileId: userProfileId }
}
// Revenue attribution endpoints
@Post('profile-id')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async getOrCreateProfileId(
@Body() dto: GetProfileIdDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ profileId: string | null }> {
const { 'user-agent': userAgent } = headers
const { pid } = dto
try {
const ip = getIPFromHeaders(headers) || reqIP || ''
const profileId = await this.analyticsService.generateProfileId(
pid,
userAgent,
ip,
)
this.logger.log(
`pid: ${pid}, profileId: ${profileId}`,
'POST /analytics/profile-id',
)
return { profileId }
} catch (error) {
this.logger.error({ error, pid }, 'Error generating profile ID')
return { profileId: null }
}
}
@Post('session-id')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async getOrCreateSessionId(
@Body() dto: GetSessionIdDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ sessionId: string | null }> {
const { 'user-agent': userAgent } = headers
const { pid } = dto
try {
const ip = getIPFromHeaders(headers) || reqIP || ''
const { psid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
this.logger.log(
`pid: ${pid}, sessionId: ${psid}`,
'POST /analytics/session-id',
)
return { sessionId: psid }
} catch (error) {
this.logger.error({ error, pid }, 'Error generating session ID')
return { sessionId: null }
}
}
}