📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
import { ApiKeyCollection } from '../api-key/api-key-collection.decorator'
import { ApiKeyAccess } from '../api-key/api-key-access.decorator'
import type { Response as ExpressResponse } from 'express'
import { SessionReplayExportService } from './session-replay-export.service'
import {
GetSessionReplayDto,
GetSessionReplaysDto,
SessionReplayExportStartDto,
SessionReplayChunkDto,
SessionReplayStartDto,
} from './dto/session-replay.dto'
import _isEmpty from 'lodash/isEmpty'
import _isArray from 'lodash/isArray'
import _map from 'lodash/map'
import _includes from 'lodash/includes'
import _size from 'lodash/size'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import dayjsTimezone from 'dayjs/plugin/timezone'
import {
Controller,
Delete,
Param,
StreamableFile,
Body,
Query,
UseGuards,
UsePipes,
ValidationPipe,
Get,
Post,
Patch,
Headers,
HttpCode,
BadRequestException,
InternalServerErrorException,
UnprocessableEntityException,
Ip,
ForbiddenException,
Response,
Header,
} from '@nestjs/common'
import { ApiTags } from '@nestjs/swagger'
import { OptionalJwtAccessTokenGuard } from '../auth/guards'
import { Auth, Public } from '../auth/decorators'
import {
AnalyticsService,
DataType,
getLowestPossibleTimeBucket,
} from './analytics.service'
import { normalizeFiltersToV1Json } from './v2/query/filters.translator'
import { VALID_PERIODS } from './decorators/validate-period.decorator'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
import { ONLINE_VISITORS_WINDOW_MINUTES } from '../common/constants'
import { AuthenticationGuard } from '../auth/guards/authentication.guard'
import { PageviewsDto } from './dto/pageviews.dto'
import { EventsDto } from './dto/events.dto'
import { GetDataDto, ChartRenderMode } from './dto/getData.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
import { DataDeletionDto } from './dto/data-deletion.dto'
import { GetVersionFiltersDto } from './dto/get-version-filters.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetPagePropertyMetaDto } from './dto/get-page-property-meta.dto'
import { GetJourneysDto } from './dto/get-journeys.dto'
import { GetJourneySessionsDto } from './dto/get-journey-sessions.dto'
import { GetFunnelsDto } from './dto/getFunnels.dto'
import { GetFunnelSessionsDto } from './dto/get-funnel-sessions.dto'
import { AppLoggerService } from '../logger/logger.service'
import { clickhouse } from '../common/integrations/clickhouse'
import { checkRateLimit, getIPDetails, getIPFromHeaders } from '../common/utils'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import { GetBotStatsDto } from './dto/get-bot-stats.dto'
import { IFunnel, IGetFunnel, IPageProperty, PerfMeasure } from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
import { GetProfilesDto } from './dto/get-profiles.dto'
import { GetProfileDto, GetProfileSessionsDto } from './dto/get-profile.dto'
import { GetProfileIdDto, GetSessionIdDto } from './dto/get-id.dto'
import { IdentifyDto } from './dto/identify.dto'
import { ErrorDto } from './dto/error.dto'
import { GetErrorsDto } from './dto/get-errors.dto'
import { GetErrorDto } from './dto/get-error.dto'
import {
GetErrorOverviewDto,
GetErrorOverviewOptions,
} from './dto/get-error-overview.dto'
import { PatchStatusDto } from './dto/patch-status.dto'
import { OutboundDto } from './dto/outbound.dto'
import { eventTransformer } from './utils/transformers'
import { enrichTrafficSource } from './utils/clickIdSources'
import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto'
import { GetOverallStatsDto } from './dto/get-overall-stats.dto'
import { GetHeartbeatStatsDto } from './dto/get-heartbeat-stats'
import { LiveVisitorsDto } from './dto/live-visitors.dto'
import { NoscriptDto } from './dto/noscript.dto'
import { GetKeywordsDto } from './dto/get-keywords.dto'
import { GSC_ALL_TIME_DAYS, GSCService } from '../project/gsc.service'
import { ExperimentService } from '../experiment/experiment.service'
import { getExperimentVariant } from '../feature-flag/evaluation'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
const DEFAULT_MEASURE = 'median'
// Silent 200 response for bots
// https://github.com/Swetrix/swetrix/issues/371
const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
return (
perf.dns <= MAX_PERFORMANCE_VALUE &&
perf.tls <= MAX_PERFORMANCE_VALUE &&
perf.conn <= MAX_PERFORMANCE_VALUE &&
perf.response <= MAX_PERFORMANCE_VALUE &&
perf.render <= MAX_PERFORMANCE_VALUE &&
perf.dom_load <= MAX_PERFORMANCE_VALUE &&
perf.page_load <= MAX_PERFORMANCE_VALUE &&
perf.ttfb <= MAX_PERFORMANCE_VALUE &&
perf.dns >= 0 &&
perf.tls >= 0 &&
perf.conn >= 0 &&
perf.response >= 0 &&
perf.render >= 0 &&
perf.dom_load >= 0 &&
perf.page_load >= 0 &&
perf.ttfb >= 0
)
}
const getPIDsArray = (pids, pid) => {
const pidsEmpty = _isEmpty(pids)
const pidEmpty = _isEmpty(pid)
if (pidsEmpty && pidEmpty)
throw new BadRequestException(
"An array of Project ID's (pids) or a Project ID (pid) has to be provided",
)
else if (!pidsEmpty && !pidEmpty)
throw new BadRequestException(
"Please provide either an array of Project ID's (pids) or a Project ID (pid), but not both",
)
else if (!pidEmpty) {
pids = JSON.stringify([pid])
}
try {
pids = JSON.parse(pids)
} catch {
throw new UnprocessableEntityException(
"Cannot process the provided array of Project ID's",
)
}
if (!_isArray(pids)) {
throw new UnprocessableEntityException(
"An array of Project ID's has to be provided as a 'pids' param",
)
}
return pids
}
const getEIDsArray = (eids, eid) => {
const eidsEmpty = _isEmpty(eids)
const eidEmpty = _isEmpty(eid)
if (eidsEmpty && eidEmpty) {
throw new BadRequestException(
"An array of Error ID's (eids) or a Error ID (eid) has to be provided",
)
}
if (!eidsEmpty && !eidEmpty) {
throw new BadRequestException(
"Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",
)
}
if (!eidsEmpty) {
return eids
}
return [eid]
}
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
'base64',
)
const getPagePath = (page?: string | string[] | null) => {
const value = Array.isArray(page) ? page[0] : page
if (!value) return null
const path = value.trim().split('#')[0].split('?')[0]
if (!path.startsWith('/') || path.startsWith('//')) return null
return path || '/'
}
const getPageFromReferrer = (referrer?: string | string[] | null) => {
const value = Array.isArray(referrer) ? referrer[0] : referrer
if (!value) return null
try {
const url = new URL(value)
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
return getPagePath(url.pathname)
} catch {
return null
}
}
@ApiTags('Analytics')
@UseGuards(OptionalJwtAccessTokenGuard, AuthenticationGuard)
@UsePipes(
new ValidationPipe({
transform: true,
}),
)
@Controller(['log', 'v1/log'])
export class AnalyticsController {
constructor(
private readonly analyticsService: AnalyticsService,
private readonly sessionReplayExportService: SessionReplayExportService,
private readonly logger: AppLoggerService,
private readonly gscService: GSCService,
private readonly experimentService: ExperimentService,
) {}
@Get()
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', ['events:read'])
async getData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
isCaptcha = false,
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
mode = ChartRenderMode.PERIODICAL,
metrics,
includeConcurrency,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}, metrics: ${metrics}`,
'GET /analytics',
)
let parsedMetrics
if (!isCaptcha) {
parsedMetrics = this.analyticsService.parseMetrics(metrics)
if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {
throw new BadRequestException(
`The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,
)
}
}
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
isCaptcha ? 'captcha' : 'pageview',
)
diff = res.diff
newTimebucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
from,
to,
newTimebucket,
period,
safeTimezone,
diff,
)
const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(
filtersQuery,
customEVFilterApplied,
isCaptcha,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
let result: any | void
if (isCaptcha) {
result = await this.analyticsService.groupCaptchaByTimeBucket(
newTimebucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
mode,
period,
)
} else {
result = await this.analyticsService.groupByTimeBucket(
newTimebucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
parsedFilters,
mode,
includeConcurrency === 'true',
)
}
if (isCaptcha) {
return {
...result,
appliedFilters: parsedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}
const customs = await this.analyticsService.getCustomEvents(
filtersQuery,
paramsData,
)
let properties: IPageProperty = {}
if (!customEVFilterApplied) {
properties = await this.analyticsService.getPageProperties(
filtersQuery,
paramsData,
)
}
let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResults>>
if (!_isEmpty(parsedMetrics)) {
meta = await this.analyticsService.getMetaResults(
pid,
parsedMetrics,
filtersQuery,
paramsData,
timezone,
period,
from,
to,
)
}
return {
...result,
customs,
properties,
appliedFilters: parsedFilters,
timeBucket: allowedTumebucketForPeriodAll,
meta,
}
}
@Get('captcha')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getCaptchaData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
) {
return this.getData(data, uid, {}, true)
}
@Get('funnel')
@Auth(true, true)
@ApiKeyAccess('funnels:read', 'query', 'pid')
async getFunnel(
@Query() data: GetFunnelsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<IGetFunnel> {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
pages,
funnelId,
filters,
} = data
const pagesArr = await this.analyticsService.getPagesArray(
pages,
funnelId,
pid,
)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/funnel')
let diff
if (period === 'all') {
const [pageviewRes, customEventRes] = await Promise.all([
this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
),
])
diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters || '[]',
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
let funnel: IFunnel[] = []
let totalPageviews: number = 0
let timeToConvert: IGetFunnel['timeToConvert']
let stepDetails = {
countries: {},
devices: {},
browsers: {},
sources: {},
campaigns: {},
pages: {},
profileTypes: {},
} as Record<string, Record<number, Record<string, number>>>
const promises = [
(async () => {
funnel = await this.analyticsService.getFunnel(
pagesArr,
params,
filtersQuery,
)
})(),
(async () => {
totalPageviews = await this.analyticsService.getTotalPageviews(
pid,
groupFrom,
groupTo,
filtersQuery,
filtersParams,
)
})(),
(async () => {
try {
stepDetails = await this.analyticsService.getFunnelStepDetails(
pagesArr,
params,
filtersQuery,
)
} catch (e) {
this.logger.error(e, 'GET /analytics/funnel - getFunnelStepDetails')
}
})(),
(async () => {
try {
timeToConvert = await this.analyticsService.getFunnelTimeToConvert(
pagesArr,
params,
filtersQuery,
)
} catch (e) {
this.logger.error(e, 'GET /analytics/funnel - getFunnelTimeToConvert')
}
})(),
]
await Promise.all(promises)
for (let i = 0; i < funnel.length; i++) {
funnel[i].topCountries = stepDetails.countries[i + 1] || {}
funnel[i].topSources = stepDetails.sources[i + 1] || {}
funnel[i].breakdowns = {
countries: stepDetails.countries[i + 1] || {},
devices: stepDetails.devices[i + 1] || {},
browsers: stepDetails.browsers[i + 1] || {},
sources: stepDetails.sources[i + 1] || {},
campaigns: stepDetails.campaigns[i + 1] || {},
pages: stepDetails.pages[i + 1] || {},
profileTypes: stepDetails.profileTypes[i + 1] || {},
}
}
return { funnel, totalPageviews, timeToConvert }
}
@Get('funnel-sessions')
@Auth(true, true)
@ApiKeyAccess('funnels:read', 'query', 'pid', [
'analytics:read',
'events:read',
'errors:read',
])
async getFunnelSessions(
@Query() data: GetFunnelSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
pages,
funnelId,
step,
filters,
dropoff,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const pagesArr = await this.analyticsService.getPagesArray(
pages,
funnelId,
pid,
)
if (step < 1 || step > pagesArr.length) {
throw new BadRequestException(
'Step must be between 1 and the number of funnel steps',
)
}
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, step: ${step}, take: ${take}, skip: ${skip}`,
'GET /analytics/funnel-sessions',
)
let diff
if (period === 'all') {
const [pageviewRes, customEventRes] = await Promise.all([
this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
),
])
diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters || '[]',
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
const isDropoff = dropoff === true || dropoff === 'true'
const sessions = await this.analyticsService.getFunnelSessionsList(
pagesArr,
params,
safeTimezone,
step,
take,
skip,
filtersQuery,
isDropoff,
)
return { sessions, take, skip }
}
@Get('meta')
@Auth(true, true)
@ApiKeyAccess('events:read', 'query', 'pid')
async getCustomEventMetadata(
@Query() data: GetCustomEventMetadata,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {
const { pid } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}`, 'GET /analytics/meta')
return this.analyticsService.getCustomEventMetadata(data)
}
@Get('property')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPagePropertyMetadata(
@Query() data: GetPagePropertyMetaDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {
const { pid } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}`, 'GET /analytics/property')
return this.analyticsService.getPagePropertyMeta(data)
}
@Get('filters')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}, type: ${type}`, 'GET /analytics/filters')
return this.analyticsService.getFilters(pid, type)
}
@Get('chart')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getChartData(
@Query() data: GetDataDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
mode = ChartRenderMode.PERIODICAL,
includeConcurrency,
} = data
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart')
const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
const result = await this.analyticsService.groupChartByTimeBucket(
timeBucket,
groupFrom,
groupTo,
filtersQuery,
paramsData,
safeTimezone,
customEVFilterApplied,
mode,
includeConcurrency === 'true',
)
return { ...result, appliedFilters: parsedFilters }
}
@Get('performance')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerfData(
@Query() data: GetDataDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
measure = DEFAULT_MEASURE,
} = data
this.analyticsService.checkIfPerfMeasureIsValid(measure)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}, measure: ${measure}`,
'GET /analytics/performance',
)
let newTimeBucket = timeBucket
let allowedTumebucketForPeriodAll
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'performance',
)
diff = res.diff
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
allowedTumebucketForPeriodAll = res.timeBucket
}
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
const result = await this.analyticsService.groupPerfByTimeBucket(
newTimeBucket,
groupFrom,
groupTo,
subQuery,
filtersQuery,
paramsData,
safeTimezone,
measure,
)
return {
...result,
appliedFilters: parsedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
}
@Get('performance/chart')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerfChartData(
@Query() data: GetDataDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
measure = DEFAULT_MEASURE,
} = data
this.analyticsService.checkIfPerfMeasureIsValid(measure)
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}, measure: ${measure}`,
'GET /analytics/performance/chart',
)
const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
const chart = await this.analyticsService.getPerfChartData(
timeBucket,
from,
to,
filtersQuery,
paramsData,
safeTimezone,
measure,
)
return { chart, appliedFilters: parsedFilters }
}
@Get('journeys')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getJourneys(
@Query() data: GetJourneysDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const steps = this.analyticsService.getSafeNumber(data.steps, 3)
const journeys = this.analyticsService.getSafeNumber(data.journeys, 20)
this.logger.log(
`pid: ${pid}, period: ${period}, steps: ${steps}, journeys: ${journeys}`,
'GET /analytics/journeys',
)
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'pageview',
)
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
true,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
let nodeDetails = []
let linkDetails = []
const [flow] = await Promise.all([
this.analyticsService.getJourneys(params, filtersQuery, steps, journeys),
(async () => {
try {
const details = await this.analyticsService.getJourneyNodeDetails(
params,
filtersQuery,
steps,
journeys,
)
nodeDetails = details.nodes
linkDetails = details.links
} catch (e) {
this.logger.error(
e,
'GET /analytics/journeys - getJourneyNodeDetails',
)
}
})(),
])
return { ...flow, nodeDetails, linkDetails, appliedFilters: parsedFilters }
}
@Get('journey-sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'analytics:read',
'events:read',
'errors:read',
])
async getJourneySessions(
@Query() data: GetJourneySessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
step,
page,
filters,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, step: ${step}, take: ${take}, skip: ${skip}`,
'GET /analytics/journey-sessions',
)
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'pageview',
)
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
)
const params = { pid, groupFrom, groupTo, ...filtersParams }
const sessions = await this.analyticsService.getJourneySessionsList(
params,
safeTimezone,
step,
page,
take,
skip,
filtersQuery,
)
return { sessions, take, skip }
}
@Get('birdseye')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
// returns overall short statistics per project
async getOverallStats(
@Query() data: GetOverallStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pids,
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
filters,
timeBucket,
includeChart,
} = data
const pidsArray = getPIDsArray(pids, pid)
this.logger.log(`pidsArray: ${pidsArray}`, 'GET /analytics/birdseye')
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
})
await Promise.all(validationPromises)
return this.analyticsService.getAnalyticsSummary(
pidsArray,
timeBucket,
period,
from,
to,
timezone,
filters,
includeChart,
)
}
@Get('performance/birdseye')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getPerformanceOverallStats(
@Query() data: GetOverallStatsDto & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pids,
pid,
period,
from,
to,
timezone = DEFAULT_TIMEZONE,
filters,
} = data
let { measure = DEFAULT_MEASURE } = data
const pidsArray = getPIDsArray(pids, pid)
this.analyticsService.checkIfPerfMeasureIsValid(measure)
if (measure === 'quantiles') {
measure = DEFAULT_MEASURE
}
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
})
await Promise.all(validationPromises)
this.logger.log(
`pidsArray: ${pidsArray}, measure: ${measure}`,
'GET /analytics/performance/birdseye',
)
return this.analyticsService.getPerformanceSummary(
pidsArray,
period,
from,
to,
timezone,
filters,
measure,
)
}
@Get('hb')
async getHeartBeatStats(
@Query() data: GetHeartbeatStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pids, pid } = data
const pidsArray = getPIDsArray(pids, pid)
this.logger.log(`pidsArray: ${pidsArray}`, 'GET /analytics/hb')
const validationPromises = _map(pidsArray, async (currentPID) => {
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
headers['x-password'],
)
})
await Promise.all(validationPromises)
const result = {}
const keyCountPromises = _map(pidsArray, async (currentPID) => {
result[currentPID] =
await this.analyticsService.getOnlineUserCount(currentPID)
})
await Promise.all(keyCountPromises).catch((reason) => {
this.logger.error(`[GET /analytics/hb] ${reason}`)
throw new InternalServerErrorException(
'An error occured while calculating heartbeat statistics',
)
})
return result
}
@Get('live-visitors')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getLiveVisitors(
@Query() queryParams: LiveVisitorsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pid } = queryParams
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}`, 'GET /analytics/live-visitors')
const since = dayjs
.utc()
.subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')
.format('YYYY-MM-DD HH:mm:ss')
const query = `
SELECT
dv,
br,
os,
cc,
psidCasted AS psid
FROM (
SELECT
any(dv) AS dv,
any(br) AS br,
any(os) AS os,
any(cc) AS cc,
toString(psid) AS psidCasted
FROM events
WHERE
pid = {pid:FixedString(12)}
AND type IN ('pageview', 'custom_event')
AND created >= {since:DateTime}
AND psid IS NOT NULL
AND psid != 0
GROUP BY psid
)
`
const { data } = await clickhouse
.query({
query,
query_params: {
pid,
since,
},
})
.then((resultSet) => resultSet.json())
return data
}
@Post('data-deletion/preview')
@HttpCode(200)
@Auth()
async getDataDeletionPreview(
@Body() body: DataDeletionDto,
@CurrentUserId() uid: string,
) {
await this.analyticsService.checkManageAccess(body.pid, uid)
return this.analyticsService.getDataDeletionPreview(
body.pid,
normalizeFiltersToV1Json(body.filters, 'traffic'),
body.from || null,
body.to || null,
)
}
@Post('data-deletion')
@HttpCode(200)
@Auth()
async deleteData(
@Body() body: DataDeletionDto,
@CurrentUserId() uid: string,
) {
await this.analyticsService.checkManageAccess(body.pid, uid)
this.logger.log(
{ uid, pid: body.pid, types: body.types },
'POST /log/data-deletion',
)
await this.analyticsService.deleteData(
body.pid,
normalizeFiltersToV1Json(body.filters, 'traffic'),
body.from || null,
body.to || null,
body.types || ['pageview', 'custom_event'],
)
return { deleted: true }
}
@Post('custom')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async logCustom(
@Body() eventsDTO: EventsDto,
@Headers() headers,
@Ip() reqIP,
) {
return this.logInteraction(eventsDTO, headers, reqIP)
}
@Post('outbound')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async logOutbound(
@Body() outboundDTO: OutboundDto,
@Headers() headers,
@Ip() reqIP,
) {
return this.logInteraction(
{ ...outboundDTO, ev: '', unique: false },
headers,
reqIP,
outboundDTO.url,
outboundDTO.title,
)
}
private async logInteraction(
eventsDTO: EventsDto,
headers,
reqIP,
outboundUrl?: string,
outboundTitle?: string | null,
) {
const isOutbound = outboundUrl !== undefined
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
eventsDTO.pid,
userAgent,
headers,
ip,
eventsDTO.ref || headers.referer || headers.referrer,
eventsDTO.pg,
'custom',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(eventsDTO, origin, ip)
if (!isOutbound && eventsDTO.unique) {
const [unique] = await this.analyticsService.generateAndStoreSessionId(
`${eventsDTO.pid}-${eventsDTO.ev}`,
userAgent,
ip,
)
if (!unique) {
throw new ForbiddenException(
'The unique option provided, while the custom event have already been created for this session',
)
}
}
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, eventsDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${eventsDTO.pid}, event: ${isOutbound ? outboundUrl : eventsDTO.ev}, pg: ${eventsDTO.pg}`,
'POST /analytics/custom',
)
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
eventsDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
eventsDTO.pid,
userAgent,
ip,
eventsDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
eventsDTO.pid,
profileId,
isNewSession,
)
enrichTrafficSource(eventsDTO)
const transformed = eventTransformer({
...(isOutbound
? { type: 'outbound' as const, url: outboundUrl, title: outboundTitle }
: { type: 'custom_event' as const, ev: eventsDTO.ev }),
psid,
sid,
profileId,
pid: eventsDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: eventsDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: eventsDTO.lc,
ref: eventsDTO.ref,
so: eventsDTO.so,
me: eventsDTO.me,
ca: eventsDTO.ca,
te: eventsDTO.te,
co: eventsDTO.co,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: eventsDTO.meta,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
if (!isOutbound)
void this.trackCustomEventExperimentExposures(
eventsDTO.pid,
eventsDTO.ev,
profileId,
transformed.created,
)
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
'Error occurred while saving the event',
)
}
return {}
}
private async trackCustomEventExperimentExposures(
pid: string,
eventName: string,
profileId: string,
created: string,
) {
try {
const experiments =
await this.experimentService.findRunningCustomEventExperiments(
pid,
eventName,
)
if (_isEmpty(experiments)) {
return
}
const exposures = []
for (const experiment of experiments) {
if (!experiment.variants || experiment.variants.length === 0) {
continue
}
const sortedVariants = [...experiment.variants].sort((a, b) =>
a.key.localeCompare(b.key),
)
const variantKey = getExperimentVariant(
experiment.id,
sortedVariants.map((variant) => ({
key: variant.key,
rolloutPercentage: variant.rolloutPercentage,
})),
profileId,
)
if (!variantKey) {
continue
}
exposures.push({
pid,
experimentId: experiment.id,
variantKey,
profileId,
created,
})
}
if (_isEmpty(exposures)) {
return
}
clickhouse
.insert({
table: 'experiment_exposures',
values: exposures,
format: 'JSONEachRow',
clickhouse_settings: { async_insert: 1 },
})
.catch((reason) => {
this.logger.warn(
{ reason },
'Failed to async insert custom event experiment exposures',
)
})
} catch (reason) {
this.logger.warn(
{ reason },
'Failed to track custom event experiment exposures',
)
}
}
@Post('hb')
@Auth(true, true)
@ApiKeyAccess('events:write', 'body', 'pid')
async heartbeat(
@Body() logDTO: PageviewsDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const { pid } = logDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
logDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
logDTO.pg,
'heartbeat',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
if (!exists || !sid) {
throw new ForbiddenException(
'The heartbeat was not saved because there is no session for this request. Please, send a pageview or custom event request first to initialise the session.',
)
}
const profileId = await this.analyticsService.generateProfileId(
pid,
userAgent,
ip,
logDTO.profileId,
)
await this.analyticsService.extendSessionTTL(psid)
await this.analyticsService.recordSessionActivity(sid, psid, pid, profileId)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'POST /analytics/hb')
return {}
}
@Post()
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async log(@Body() logDTO: PageviewsDto, @Headers() headers, @Ip() reqIP) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
logDTO.pid,
userAgent,
headers,
ip,
logDTO.ref || headers.referer || headers.referrer,
logDTO.pg,
'pageview',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(logDTO, origin, ip)
const [unique, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
logDTO.pid,
userAgent,
ip,
logDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
logDTO.pid,
profileId,
unique,
)
if (!unique && logDTO.unique) {
throw new ForbiddenException(
'The event was not saved because it was not unique while unique only param is provided',
)
}
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, logDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(`pid: ${logDTO.pid}, pg: ${logDTO.pg}`, 'POST /analytics')
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
enrichTrafficSource(logDTO)
const transformed = eventTransformer({
type: 'pageview',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
title: logDTO.title,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: logDTO.lc,
ref: logDTO.ref,
so: logDTO.so,
me: logDTO.me,
ca: logDTO.ca,
te: logDTO.te,
co: logDTO.co,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: logDTO.meta,
})
let perfTransformed = null
if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {
const {
dns,
tls,
conn,
response,
render,
dom_load: domLoad,
page_load: pageLoad,
ttfb,
} = logDTO.perf
perfTransformed = eventTransformer({
type: 'performance',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
dns,
tls,
conn,
response,
render,
domLoad,
pageLoad,
ttfb,
})
}
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
if (!_isEmpty(perfTransformed)) {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [perfTransformed],
clickhouse_settings: { async_insert: 1 },
})
}
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
'Error occured while saving the log data',
)
}
return {}
}
// Fallback for logging pageviews for users with JavaScript disabled
// Returns 1x1 transparent gif
@Get('noscript')
@Header('Cross-Origin-Resource-Policy', 'cross-origin')
@Public()
async noscript(
@Query() data: NoscriptDto,
@Headers() headers,
@Ip() reqIP,
@Response() res,
) {
const { 'user-agent': userAgent, origin } = headers
const { pid } = data
const referrer = headers.referer || headers.referrer
const pg =
getPagePath(data.pg) || getPageFromReferrer(referrer) || undefined
const logDTO: PageviewsDto = { pid, pg }
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
pid,
userAgent,
headers,
ip,
referrer,
pg ?? null,
'noscript',
)
if (botResult.isBot) {
res.writeHead(200, { 'Content-Type': 'image/gif' })
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
const project = await this.analyticsService.validate(logDTO, origin, ip)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
)
// For noscript requests, we generate an anonymous profile ID since
// user-supplied profileId is not available without JavaScript
const profileId = await this.analyticsService.generateProfileId(
logDTO.pid,
userAgent,
ip,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
logDTO.pid,
profileId,
isNewSession,
)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, null)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(`pid: ${pid}, pg: ${pg}`, 'GET /analytics/noscript')
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const transformed = eventTransformer({
type: 'pageview',
psid,
sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: null,
ref: null,
so: null,
me: null,
ca: null,
te: null,
co: null,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
meta: null,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
} catch (e) {
this.logger.error(e)
}
res.writeHead(200, { 'Content-Type': 'image/gif' })
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
@Get('sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
'replays:read',
])
async getSessions(
@Query() data: GetSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
const sessionEvent = data.sessionEvent || 'traffic'
const sessionDataType =
sessionEvent === 'performance'
? DataType.PERFORMANCE
: sessionEvent === 'error'
? DataType.ERRORS
: DataType.ANALYTICS
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/sessions',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
filters,
sessionDataType,
sessionEvent !== 'traffic',
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
sessionEvent === 'traffic'
? this.analyticsService.getAnalyticsEventType(customEVFilterApplied)
: sessionEvent,
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const sessions = await this.analyticsService.getSessionsList(
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
sessionEvent,
)
return { sessions, appliedFilters, take, skip }
}
@Get('session')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
'replays:read',
])
async getSession(
@Query() data: GetSessionDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, psid, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'GET /analytics/session')
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const result = await this.analyticsService.getSessionDetails(
pid,
psid,
safeTimezone,
)
return result
}
@Get('custom-events')
@Auth(true, true)
@ApiKeyAccess('events:read', 'query', 'pid')
async getCustomEvents(
@Query() data: GetCustomEventsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
timeBucket,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
customEvents,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}, customEvents: ${customEvents}`,
'GET /analytics/custom-events',
)
let newTimeBucket = timeBucket
let diff
let timeBucketForAllTime
if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'custom_event',
)
newTimeBucket = _includes(res.timeBucket, timeBucket)
? timeBucket
: res.timeBucket[0]
diff = res.diff
timeBucketForAllTime = res.timeBucket
}
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
// customEvents comes as a JSON.stringified array from the frontend
let customEventsList: string[] = []
try {
const parsed = JSON.parse(customEvents || '[]')
customEventsList = Array.isArray(parsed) ? parsed : []
} catch {
customEventsList = []
}
if (customEventsList.length === 0) {
return {
chart: {},
appliedFilters: parsedFilters,
timeBucket: timeBucketForAllTime,
}
}
const result: any = await this.analyticsService.groupCustomEVByTimeBucket(
newTimeBucket,
groupFrom,
groupTo,
filtersQuery,
paramsData,
safeTimezone,
customEventsList,
)
return {
...result,
appliedFilters: parsedFilters,
timeBucket: timeBucketForAllTime,
}
}
@Get('errors-filters')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrorsFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, type: ${type}`,
'GET /analytics/errors-filters',
)
return this.analyticsService.getErrorsFilters(pid, type)
}
@Get('filters/versions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getVersionFilters(
@Query() data: GetVersionFiltersDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, type, column } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, type: ${type}, column: ${column}`,
'GET /analytics/filters/versions',
)
return this.analyticsService.getVersionFilters(pid, type, column)
}
@Get('bot-stats')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getBotStats(
@Query() data: GetBotStatsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period = '30d' } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}`,
'GET /analytics/bot-stats',
)
return this.analyticsService.getBotStats(pid, period)
}
@Post('error')
@ApiKeyCollection()
@ApiKeyAccess('errors:write', 'body', 'pid')
@Public()
async logError(@Body() errorDTO: ErrorDto, @Headers() headers, @Ip() reqIP) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
const botResult = await this.analyticsService.checkBot(
errorDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
errorDTO.pg,
'error',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(errorDTO, origin, ip)
const [isNewSession, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
errorDTO.pid,
userAgent,
ip,
)
const profileId = await this.analyticsService.generateProfileId(
errorDTO.pid,
userAgent,
ip,
errorDTO.profileId,
)
await this.analyticsService.recordSessionActivity(
sid,
psid,
errorDTO.pid,
profileId,
isNewSession,
)
const {
city,
region,
regionCode,
country,
isp,
organization,
userType,
connectionType,
} = getIPDetails(ip, errorDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${errorDTO.pid}, pg: ${errorDTO.pg}, name: ${errorDTO.name}, message: ${errorDTO.message}`,
'POST /analytics/error',
)
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
const { name, message, lineno, colno, filename, stackTrace, meta } =
errorDTO
const transformed = eventTransformer({
type: 'error',
psid,
sid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
pid: errorDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: errorDTO.pg,
dv: deviceType,
br: browserName,
brv: browserVersion,
os: osName,
osv: osVersion,
lc: errorDTO.lc,
cc: country,
rg: region,
rgc: regionCode,
ct: city,
isp,
og: organization,
ut: userType,
ctp: connectionType,
name,
message,
lineno,
colno,
filename,
stackTrace,
meta,
})
try {
await clickhouse.insert({
table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
})
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
'Error occured while saving the custom event',
)
}
return {}
}
// Update error(s) status
@Patch('error-status')
@Auth(true, true)
@ApiKeyAccess('errors:write', 'body', 'pid')
async patchStatus(
@Body() statusDTO: PatchStatusDto,
@CurrentUserId() uid: string,
@Headers() headers,
@Ip() reqIP,
) {
const { pid, eid, eids: unprocessedEids, status } = statusDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'error-status', 100, 1800)
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const eids = getEIDsArray(unprocessedEids, eid)
await this.analyticsService.validateEIDs(eids, pid)
return this.analyticsService.updateEIDStatus(eids, status, pid)
}
@Get('errors')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrors(
@Query() data: GetErrorsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
options,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of errors to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/errors',
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ERRORS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const errors = await this.analyticsService.getErrorsList(
options,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
)
return { errors, appliedFilters, take, skip }
}
@Get('get-error')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getError(
@Query() data: GetErrorDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
timezone = DEFAULT_TIMEZONE,
eid,
period,
from,
to,
timeBucket,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, eid: ${eid}, period: ${period}`,
'GET /analytics/get-error',
)
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const result = await this.analyticsService.getErrorDetails(
pid,
eid,
safeTimezone,
groupFromUTC,
groupToUTC,
newTimeBucket,
)
return result
}
@Get('error-overview')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid')
async getErrorOverview(
@Query() data: GetErrorOverviewDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
timeBucket,
options,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, period: ${period}`,
'GET /analytics/error-overview',
)
let parsedOptions: GetErrorOverviewOptions = {}
if (typeof options === 'string') {
try {
parsedOptions = JSON.parse(options)
} catch {
// Ignore parse errors
}
} else {
parsedOptions = options || {}
}
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters,
DataType.ERRORS,
true,
)
const [sessionFiltersQuery, sessionFiltersParams] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
return this.analyticsService.getErrorOverview(
pid,
filtersQuery,
paramsData,
safeTimezone,
groupFromUTC,
groupToUTC,
newTimeBucket,
parsedOptions.showResolved || false,
sessionFiltersQuery,
sessionFiltersParams,
)
}
@Get('error-sessions')
@Auth(true, true)
@ApiKeyAccess('errors:read', 'query', 'pid', ['analytics:read'])
async getErrorSessions(
@Query() data: GetErrorDto & { take?: number; skip?: number },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
eid,
period,
from,
to,
timeBucket,
filters,
timezone = DEFAULT_TIMEZONE,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 10)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 50) {
throw new BadRequestException(
'The maximum number of sessions to return is 50',
)
}
this.logger.log(
`pid: ${pid}, eid: ${eid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/error-sessions',
)
const [filtersQuery, filtersParams] = this.analyticsService.getFiltersQuery(
filters,
DataType.ERRORS,
true,
)
let newTimeBucket = timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
'error',
)
newTimeBucket = res.timeBucket[0]
diff = res.diff
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
newTimeBucket,
period,
safeTimezone,
diff,
)
return this.analyticsService.getErrorAffectedSessions(
pid,
eid,
groupFromUTC,
groupToUTC,
take,
skip,
filtersQuery,
filtersParams,
)
}
@Get('profiles')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
])
async getProfiles(
@Query() data: GetProfilesDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
profileType = 'all',
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of profiles to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/profiles',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
this.analyticsService.getAnalyticsEventType(customEVFilterApplied),
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const profiles = await this.analyticsService.getProfilesList(
pid,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
profileType,
customEVFilterApplied,
)
return { profiles, appliedFilters, take, skip }
}
@Get('profile')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
])
async getProfile(
@Query() data: GetProfileDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, profileId, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, profileId: ${profileId}`,
'GET /analytics/profile',
)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const [details, topPages, activityCalendar] = await Promise.all([
this.analyticsService.getProfileDetails(pid, profileId, safeTimezone),
this.analyticsService.getProfileTopPages(pid, profileId),
this.analyticsService.getProfileActivityCalendar(pid, profileId),
])
return {
...details,
topPages,
activityCalendar,
}
}
@Get('profile/sessions')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid', [
'events:read',
'errors:read',
])
async getProfileSessions(
@Query() data: GetProfileSessionsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const {
pid,
profileId,
period,
from,
to,
filters,
timezone = DEFAULT_TIMEZONE,
} = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of sessions to return is 150',
)
}
this.logger.log(
`pid: ${pid}, profileId: ${profileId}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/profile/sessions',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
this.analyticsService.getAnalyticsEventType(customEVFilterApplied),
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const sessions = await this.analyticsService.getProfileSessionsList(
pid,
profileId,
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
)
return { sessions, appliedFilters, take, skip }
}
/**
* Links the visitor's current anonymous profile to an identified profile
* derived from the user ID supplied by the site (e.g. after log in). Events
* previously recorded for the anonymous profile get attributed to the
* identified profile at query time; the tracker stamps all subsequent
* events with the supplied profileId directly.
*
* Optional traits (email, plan, ...) are stored against the identified
* profile and shown on its dashboard page.
*/
@Post('identify')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async identify(
@Body() dto: IdentifyDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ profileId: string } | typeof BOT_RESPONSE> {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'identify', 120, 60)
await checkRateLimit(dto.pid, 'identify', 2000, 60)
const botResult = await this.analyticsService.checkBot(
dto.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
null,
'identify',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const profileId = this.analyticsService.validateUserSuppliedProfileId(
dto.profileId,
)
await this.analyticsService.validate(dto, origin, ip)
const userProfileId = await this.analyticsService.generateProfileId(
dto.pid,
userAgent,
ip,
profileId,
)
if (!_isEmpty(dto.traits)) {
await this.analyticsService.saveProfileTraits(
dto.pid,
userProfileId,
dto.traits,
)
}
const anonProfileId = await this.analyticsService.generateProfileId(
dto.pid,
userAgent,
ip,
)
const linked = await this.analyticsService.linkProfiles(
dto.pid,
anonProfileId,
userProfileId,
)
// Flip the visitor's current session (if any) to the identified profile.
// Skipped when the anonymous profile is already linked to a different
// identified profile (e.g. a second account on a shared device) - the
// session stays with the current identity until new events re-stamp it.
if (linked) {
const { exists, psid, sid } = await this.analyticsService.getSessionId(
dto.pid,
userAgent,
ip,
)
if (exists && sid) {
await this.analyticsService.recordSessionActivity(
sid,
psid,
dto.pid,
userProfileId,
)
}
}
this.logger.log(
`pid: ${dto.pid}, profileId: ${userProfileId}`,
'POST /analytics/identify',
)
return { profileId: userProfileId }
}
@Post('profile-id')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async getOrCreateProfileId(
@Body() dto: GetProfileIdDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ profileId: string | null }> {
const { 'user-agent': userAgent } = headers
const { pid } = dto
try {
const ip = getIPFromHeaders(headers) || reqIP || ''
const profileId = await this.analyticsService.generateProfileId(
pid,
userAgent,
ip,
)
this.logger.log(
`pid: ${pid}, profileId: ${profileId}`,
'POST /analytics/profile-id',
)
return { profileId }
} catch (error) {
this.logger.error({ error, pid }, 'Error generating profile ID')
return { profileId: null }
}
}
@Post('session-id')
@ApiKeyCollection()
@ApiKeyAccess('events:write', 'body', 'pid')
@Public()
async getOrCreateSessionId(
@Body() dto: GetSessionIdDto,
@Headers() headers,
@Ip() reqIP,
): Promise<{ sessionId: string | null }> {
const { 'user-agent': userAgent } = headers
const { pid } = dto
try {
const ip = getIPFromHeaders(headers) || reqIP || ''
const { psid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
this.logger.log(
`pid: ${pid}, sessionId: ${psid}`,
'POST /analytics/session-id',
)
return { sessionId: psid }
} catch (error) {
this.logger.error({ error, pid }, 'Error generating session ID')
return { sessionId: null }
}
}
@Get('keywords')
@Auth(true, true)
@ApiKeyAccess('analytics:read', 'query', 'pid')
async getKeywords(
@Query() data: GetKeywordsDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/keywords')
const diff = period === 'all' ? GSC_ALL_TIME_DAYS : undefined
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
null,
period,
safeTimezone,
diff,
)
const keywords = await this.gscService.getKeywords(pid, groupFrom, groupTo)
return { keywords }
}
@Post('session-replay/start')
@ApiKeyCollection()
@ApiKeyAccess('replays:write', 'body', 'pid')
@Public()
async startSessionReplay(
@Body() replayDTO: SessionReplayStartDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'session-replay-start', 120, 60)
await checkRateLimit(replayDTO.pid, 'session-replay-start', 2000, 60)
const botResult = await this.analyticsService.checkBot(
replayDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
replayDTO.pg,
'session_replay',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(replayDTO, origin, ip)
const { country } = getIPDetails(ip, replayDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${replayDTO.pid}, replayId: ${replayDTO.replayId}`,
'POST /analytics/session-replay/start',
)
return this.analyticsService.startSessionReplay(
project,
replayDTO.pid,
replayDTO.replayId,
replayDTO.privacy,
userAgent,
ip,
replayDTO.profileId,
)
}
@Post('session-replay/chunk')
@ApiKeyCollection()
@ApiKeyAccess('replays:write', 'body', 'pid')
@Public()
async uploadSessionReplayChunk(
@Body() replayDTO: SessionReplayChunkDto,
@Headers() headers,
@Ip() reqIP,
) {
const { 'user-agent': userAgent, origin } = headers
const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'session-replay-chunk', 600, 60)
await checkRateLimit(replayDTO.pid, 'session-replay-chunk', 5000, 60)
const botResult = await this.analyticsService.checkBot(
replayDTO.pid,
userAgent,
headers,
ip,
headers.referer || headers.referrer,
replayDTO.pg,
'session_replay',
)
if (botResult.isBot) {
return BOT_RESPONSE
}
const project = await this.analyticsService.validate(replayDTO, origin, ip)
const { country } = getIPDetails(ip, replayDTO.tz)
this.analyticsService.checkCountryBlacklist(project, country)
this.logger.log(
`pid: ${replayDTO.pid}, replayId: ${replayDTO.replayId}, chunkIndex: ${replayDTO.chunkIndex}`,
'POST /analytics/session-replay/chunk',
)
return this.analyticsService.storeSessionReplayChunk(
project,
replayDTO.pid,
replayDTO.replayId,
replayDTO.privacy,
replayDTO.chunkIndex,
replayDTO.events,
userAgent,
ip,
)
}
@Get('session-replays')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'query', 'pid')
async getSessionReplays(
@Query() data: GetSessionReplaysDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
if (take > 150) {
throw new BadRequestException(
'The maximum number of session replays to return is 150',
)
}
this.logger.log(
`pid: ${pid}, period: ${period}, take: ${take}, skip: ${skip}`,
'GET /analytics/session-replays',
)
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(
normalizeFiltersToV1Json(filters, 'traffic'),
DataType.ANALYTICS,
)
let timeBucket
let diff
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
['pageview', 'custom_event', 'error'] as const,
)
timeBucket = res.timeBucket[0]
diff = res.diff
} else {
timeBucket = getLowestPossibleTimeBucket(period, from, to)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
period,
safeTimezone,
diff,
)
const paramsData = {
params: {
pid,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...filtersParams,
},
}
const replays = await this.analyticsService.getSessionReplaysList(
filtersQuery,
paramsData,
safeTimezone,
take,
skip,
customEVFilterApplied,
)
return { replays, appliedFilters, take, skip }
}
@Get('session-replay')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'query', 'pid')
async getSessionReplay(
@Query() data: GetSessionReplayDto,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'GET /analytics/session-replay',
)
return this.analyticsService.getSessionReplay(pid, psid, replayId)
}
@Delete('session-replay')
@Auth(true, true)
@ApiKeyAccess('replays:write', 'query', 'pid')
async deleteSessionReplay(
@Query() data: GetSessionReplayDto,
@CurrentUserId() uid: string,
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkManageAccess(pid, uid)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'DELETE /analytics/session-replay',
)
return this.analyticsService.deleteSessionReplay(pid, psid, replayId)
}
@Post('session-replay/export')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'body', 'pid')
async startSessionReplayExport(
@Body() data: SessionReplayExportStartDto,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string; 'user-agent'?: string },
@Ip() requestIp: string,
) {
const { pid, psid, replayId } = data
await this.analyticsService.checkProjectAccess(
pid,
uid,
headers['x-password'],
)
const ip = getIPFromHeaders(headers) || requestIp || ''
await checkRateLimit(uid || ip || pid, 'session-replay-export-user', 5, 60)
await checkRateLimit(pid, 'session-replay-export-project', 20, 60)
this.logger.log(
`pid: ${pid}, psid: ${psid}, replayId: ${replayId || 'latest'}`,
'POST /analytics/session-replay/export',
)
return this.sessionReplayExportService.startExport(pid, psid, replayId)
}
@Get('session-replay/export/:exportId')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'export', 'exportId')
async getSessionReplayExport(
@Param('exportId') exportId: string,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string },
) {
const exportState =
await this.sessionReplayExportService.assertExportAccess(exportId)
await this.analyticsService.checkProjectAccess(
exportState.pid,
uid,
headers['x-password'],
)
return this.sessionReplayExportService.getExport(exportId)
}
@Get('session-replay/export/:exportId/download')
@Auth(true, true)
@ApiKeyAccess('replays:read', 'export', 'exportId')
async downloadSessionReplayExport(
@Param('exportId') exportId: string,
@CurrentUserId() uid: string | null,
@Headers() headers: { 'x-password'?: string },
@Response({ passthrough: true }) response: ExpressResponse,
): Promise<StreamableFile> {
const exportState =
await this.sessionReplayExportService.assertExportAccess(exportId)
await this.analyticsService.checkProjectAccess(
exportState.pid,
uid,
headers['x-password'],
)
return this.sessionReplayExportService.downloadExport(exportId, response)
}
}